From 55dcf41ed790d83054d2fd8bd036d9784f4c8590 Mon Sep 17 00:00:00 2001 From: Reese Date: Mon, 8 Dec 2025 07:16:54 +0000 Subject: [PATCH 01/55] Initial CLI/proxy wrapper PoC Gets remove MCP tools and exposes as own, allows setting user workspace/customisation in outside of .env (and affect server in-process) --- ctxce-cli/bin/ctxce.js | 7 + ctxce-cli/package-lock.json | 1091 ++++++++++++++++++++++++++++++++ ctxce-cli/package.json | 20 + ctxce-cli/src/cli.js | 31 + ctxce-cli/src/indexerClient.js | 30 + ctxce-cli/src/mcpServer.js | 201 ++++++ 6 files changed, 1380 insertions(+) create mode 100644 ctxce-cli/bin/ctxce.js create mode 100644 ctxce-cli/package-lock.json create mode 100644 ctxce-cli/package.json create mode 100644 ctxce-cli/src/cli.js create mode 100644 ctxce-cli/src/indexerClient.js create mode 100644 ctxce-cli/src/mcpServer.js diff --git a/ctxce-cli/bin/ctxce.js b/ctxce-cli/bin/ctxce.js new file mode 100644 index 00000000..d4fe0d56 --- /dev/null +++ b/ctxce-cli/bin/ctxce.js @@ -0,0 +1,7 @@ +#!/usr/bin/env node +import { runCli } from "../src/cli.js"; + +runCli().catch((err) => { + console.error("[ctxce] Fatal error:", err && err.stack ? err.stack : err); + process.exit(1); +}); diff --git a/ctxce-cli/package-lock.json b/ctxce-cli/package-lock.json new file mode 100644 index 00000000..16b6f8a1 --- /dev/null +++ b/ctxce-cli/package-lock.json @@ -0,0 +1,1091 @@ +{ + "name": "@context-engine/cli", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@context-engine/cli", + "version": "0.0.1", + "dependencies": { + "@modelcontextprotocol/sdk": "^1.24.3", + "zod": "^3.25.0" + }, + "bin": { + "ctxce": "bin/ctxce.js" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.24.3.tgz", + "integrity": "sha512-YgSHW29fuzKKAHTGe9zjNoo+yF8KaQPzDC2W9Pv41E7/57IfY+AMGJ/aDFlgTLcVVELoggKE4syABCE75u3NCw==", + "license": "MIT", + "dependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.0.1", + "express-rate-limit": "^7.5.0", + "jose": "^6.1.1", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/body-parser": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.1.tgz", + "integrity": "sha512-nfDwkulwiZYQIGwxdy0RUmowMhKcFVcYXUU7m4QlKYim1rUtg83xm2yjZ40QjDuc291AJjjeSc9b++AWHSgSHw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.0", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", + "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "7.5.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-7.5.1.tgz", + "integrity": "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw==", + "license": "MIT", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.1.3.tgz", + "integrity": "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", + "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.0.tgz", + "integrity": "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "mime-types": "^3.0.1", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/serve-static": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", + "license": "MIT", + "dependencies": { + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.0", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.0.tgz", + "integrity": "sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25 || ^4" + } + } + } +} diff --git a/ctxce-cli/package.json b/ctxce-cli/package.json new file mode 100644 index 00000000..5342dd50 --- /dev/null +++ b/ctxce-cli/package.json @@ -0,0 +1,20 @@ +{ + "name": "@context-engine/cli", + "version": "0.0.1", + "private": true, + "description": "Minimal MCP stdio proxy for Context-Engine indexer", + "bin": { + "ctxce": "bin/ctxce.js" + }, + "type": "module", + "scripts": { + "start": "node bin/ctxce.js" + }, + "dependencies": { + "@modelcontextprotocol/sdk": "^1.24.3", + "zod": "^3.25.0" + }, + "engines": { + "node": ">=18.0.0" + } +} diff --git a/ctxce-cli/src/cli.js b/ctxce-cli/src/cli.js new file mode 100644 index 00000000..0301ea49 --- /dev/null +++ b/ctxce-cli/src/cli.js @@ -0,0 +1,31 @@ +// CLI entrypoint for ctxce + +import process from "node:process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runMcpServer } from "./mcpServer.js"; + +export async function runCli() { + const argv = process.argv.slice(2); + const cmd = argv[0]; + + if (cmd === "mcp-serve") { + // TODO: add proper argument parsing; PoC uses cwd as workspace + const workspace = process.cwd(); + const indexerUrl = process.env.CTXCE_INDEXER_URL || "http://localhost:8003/mcp"; + + // eslint-disable-next-line no-console + console.error(`[ctxce] Starting MCP bridge: workspace=${workspace}, indexerUrl=${indexerUrl}`); + await runMcpServer({ workspace, indexerUrl }); + return; + } + + // Default help + const __filename = fileURLToPath(import.meta.url); + const __dirname = path.dirname(__filename); + const binName = "ctxce"; + + // eslint-disable-next-line no-console + console.error(`Usage: ${binName} mcp-serve`); + process.exit(1); +} diff --git a/ctxce-cli/src/indexerClient.js b/ctxce-cli/src/indexerClient.js new file mode 100644 index 00000000..208cfe3e --- /dev/null +++ b/ctxce-cli/src/indexerClient.js @@ -0,0 +1,30 @@ +// Minimal JSON-RPC over HTTP client for the remote Context-Engine MCP indexer + +const DEFAULT_INDEXER_URL = process.env.CTXCE_INDEXER_URL || "http://localhost:8003/mcp"; + +export class IndexerClient { + constructor(url) { + this.url = url || DEFAULT_INDEXER_URL; + } + + async call(method, params) { + const id = Date.now().toString() + Math.random().toString(16).slice(2); + const body = { jsonrpc: "2.0", id, method, params }; + + const res = await fetch(this.url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + throw new Error(`[indexer] HTTP ${res.status}: ${await res.text()}`); + } + + const json = await res.json(); + if (json.error) { + throw json.error; + } + return json.result; + } +} diff --git a/ctxce-cli/src/mcpServer.js b/ctxce-cli/src/mcpServer.js new file mode 100644 index 00000000..ea34b31f --- /dev/null +++ b/ctxce-cli/src/mcpServer.js @@ -0,0 +1,201 @@ +// MCP stdio server implemented using the official MCP TypeScript SDK. +// Acts as a low-level proxy for tools, forwarding tools/list and tools/call +// to the remote qdrant-indexer MCP server while adding a local `ping` tool. + +import process from "node:process"; +import fs from "node:fs"; +import path from "node:path"; +import { execSync } from "node:child_process"; +import { Server } from "@modelcontextprotocol/sdk/server/index.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js"; + +export async function runMcpServer(options) { + const workspace = options.workspace || process.cwd(); + const indexerUrl = options.indexerUrl; + + const config = loadConfig(workspace); + const defaultCollection = + config && typeof config.default_collection === "string" + ? config.default_collection + : null; + + // eslint-disable-next-line no-console + console.error( + `[ctxce] MCP low-level stdio bridge starting: workspace=${workspace}, indexerUrl=${indexerUrl}`, + ); + + if (defaultCollection) { + // eslint-disable-next-line no-console + console.error( + `[ctxce] Using default collection from ctx_config.json: ${defaultCollection}`, + ); + } + + // High-level MCP client for the remote HTTP /mcp indexer + const clientTransport = new StreamableHTTPClientTransport(indexerUrl); + const client = new Client( + { + name: "ctx-context-engine-bridge-http-client", + version: "0.0.1", + }, + { + capabilities: { + tools: {}, + resources: {}, + prompts: {}, + }, + }, + ); + + try { + await client.connect(clientTransport); + } catch (err) { + // eslint-disable-next-line no-console + console.error("[ctxce] Failed to connect MCP HTTP client to indexer:", err); + throw err; + } + + const server = new Server( + { + name: "ctx-context-engine-bridge", + version: "0.0.1", + }, + { + capabilities: { + tools: {}, + }, + }, + ); + + // tools/list → fetch tools from remote indexer and append local ping tool + server.setRequestHandler(ListToolsRequestSchema, async () => { + let remote; + try { + remote = await client.listTools(); + } catch (err) { + // eslint-disable-next-line no-console + console.error("[ctxce] Error calling remote tools/list:", err); + return { tools: [buildPingTool()] }; + } + + // eslint-disable-next-line no-console + console.error("[ctxce] tools/list remote result:", JSON.stringify(remote)); + + const tools = Array.isArray(remote?.tools) ? remote.tools.slice() : []; + tools.push(buildPingTool()); + return { tools }; + }); + + // tools/call → handle ping locally, everything else is proxied to indexer + server.setRequestHandler(CallToolRequestSchema, async (request) => { + const params = request.params || {}; + const name = params.name; + let args = params.arguments; + + if (name === "ping") { + const branch = detectGitBranch(workspace); + const text = args && typeof args.text === "string" ? args.text : "pong"; + const suffix = branch ? ` (branch=${branch})` : ""; + return { + content: [ + { + type: "text", + text: `${text}${suffix}`, + }, + ], + }; + } + + // Inject default collection when not explicitly provided and arguments + // are an object (indexer tools accept a collection parameter). + if ( + defaultCollection && + (args === undefined || args === null || typeof args === "object") + ) { + const obj = args && typeof args === "object" ? { ...args } : {}; + if ( + !Object.prototype.hasOwnProperty.call(obj, "collection") || + obj.collection === undefined || + obj.collection === null || + obj.collection === "" + ) { + obj.collection = defaultCollection; + } + args = obj; + } + + const result = await client.callTool({ + name, + arguments: args, + }); + return result; + }); + + const transport = new StdioServerTransport(); + await server.connect(transport); +} + +function loadConfig(startDir) { + try { + let dir = startDir; + for (let i = 0; i < 5; i += 1) { + const cfgPath = path.join(dir, "ctx_config.json"); + if (fs.existsSync(cfgPath)) { + try { + const raw = fs.readFileSync(cfgPath, "utf8"); + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === "object") { + return parsed; + } + } catch (err) { + // eslint-disable-next-line no-console + console.error("[ctxce] Failed to parse ctx_config.json:", err); + return null; + } + } + const parent = path.dirname(dir); + if (!parent || parent === dir) { + break; + } + dir = parent; + } + } catch (err) { + // eslint-disable-next-line no-console + console.error("[ctxce] Error while loading ctx_config.json:", err); + } + return null; +} + +function buildPingTool() { + return { + name: "ping", + description: "Basic ping tool exposed by the ctx bridge", + inputSchema: { + type: "object", + properties: { + text: { + type: "string", + description: "Optional text to echo back.", + }, + }, + required: [], + }, + }; +} + +function detectGitBranch(workspace) { + try { + const out = execSync("git rev-parse --abbrev-ref HEAD", { + cwd: workspace, + stdio: ["ignore", "pipe", "ignore"], + }); + const name = out.toString("utf8").trim(); + return name || null; + } catch { + return null; + } +} + From 567c866b95e914fa9b29ef8d58fb7e5ddcb08609 Mon Sep 17 00:00:00 2001 From: Reese Date: Mon, 8 Dec 2025 07:51:38 +0000 Subject: [PATCH 02/55] Introduces session defaults to indexer calls Enables the indexer to apply per-session defaults for collection, mode, and other parameters. This change introduces a `set_session_defaults` tool to allow setting defaults that persist per connection. It also modifies `repo_search` to resolve collection and related hints from per-connection defaults, token-based defaults, and environment variables. It ensures that subsequent calls to the indexer within the same session will use these defaults, improving usability and reducing the need for repetitive parameter passing. --- ctxce-cli/src/mcpServer.js | 55 ++++++++++++++++++------ scripts/mcp_indexer_server.py | 80 +++++++++++++++++++++++++++++------ 2 files changed, 107 insertions(+), 28 deletions(-) diff --git a/ctxce-cli/src/mcpServer.js b/ctxce-cli/src/mcpServer.js index ea34b31f..d73ef6f1 100644 --- a/ctxce-cli/src/mcpServer.js +++ b/ctxce-cli/src/mcpServer.js @@ -21,6 +21,10 @@ export async function runMcpServer(options) { config && typeof config.default_collection === "string" ? config.default_collection : null; + const defaultMode = + config && typeof config.default_mode === "string" ? config.default_mode : null; + const defaultUnder = + config && typeof config.default_under === "string" ? config.default_under : null; // eslint-disable-next-line no-console console.error( @@ -58,7 +62,39 @@ export async function runMcpServer(options) { throw err; } - const server = new Server( + // Derive a simple session identifier for this bridge process. In the + // future this can be made user-aware (e.g. from auth), but for now we + // keep it deterministic per workspace to help the indexer reuse + // session-scoped defaults. + const sessionId = + process.env.CTXCE_SESSION_ID || `ctxce-${Buffer.from(workspace).toString("hex").slice(0, 24)}`; + + // Best-effort: inform the indexer of default collection and session. + // If this fails we still proceed, falling back to per-call injection. + const defaultsPayload = { session: sessionId }; + if (defaultCollection) { + defaultsPayload.collection = defaultCollection; + } + if (defaultMode) { + defaultsPayload.mode = defaultMode; + } + if (defaultUnder) { + defaultsPayload.under = defaultUnder; + } + + if (Object.keys(defaultsPayload).length > 1) { + try { + await client.callTool({ + name: "set_session_defaults", + arguments: defaultsPayload, + }); + } catch (err) { + // eslint-disable-next-line no-console + console.error("[ctxce] Failed to call set_session_defaults on indexer:", err); + } + } + + const server = new Server( // TODO: marked as depreciated { name: "ctx-context-engine-bridge", version: "0.0.1", @@ -109,20 +145,11 @@ export async function runMcpServer(options) { }; } - // Inject default collection when not explicitly provided and arguments - // are an object (indexer tools accept a collection parameter). - if ( - defaultCollection && - (args === undefined || args === null || typeof args === "object") - ) { + // Attach session id so the indexer can apply per-session defaults. + if (sessionId && (args === undefined || args === null || typeof args === "object")) { const obj = args && typeof args === "object" ? { ...args } : {}; - if ( - !Object.prototype.hasOwnProperty.call(obj, "collection") || - obj.collection === undefined || - obj.collection === null || - obj.collection === "" - ) { - obj.collection = defaultCollection; + if (!Object.prototype.hasOwnProperty.call(obj, "session")) { + obj.session = sessionId; } args = obj; } diff --git a/scripts/mcp_indexer_server.py b/scripts/mcp_indexer_server.py index 8f3d879d..5bcf6843 100644 --- a/scripts/mcp_indexer_server.py +++ b/scripts/mcp_indexer_server.py @@ -1644,11 +1644,14 @@ async def qdrant_index( @mcp.tool() async def set_session_defaults( collection: Any = None, + mode: Any = None, + under: Any = None, + language: Any = None, session: Any = None, ctx: Context = None, **kwargs, ) -> Dict[str, Any]: - """Set defaults (e.g., collection) for subsequent calls. + """Set defaults (e.g., collection, mode, under) for subsequent calls. Behavior: - If request Context is available, persist defaults per-connection so later calls on @@ -1660,6 +1663,12 @@ async def set_session_defaults( if _extra: if (collection is None or (isinstance(collection, str) and collection.strip() == "")) and _extra.get("collection") is not None: collection = _extra.get("collection") + if (mode is None or (isinstance(mode, str) and str(mode).strip() == "")) and _extra.get("mode") is not None: + mode = _extra.get("mode") + if (under is None or (isinstance(under, str) and str(under).strip() == "")) and _extra.get("under") is not None: + under = _extra.get("under") + if (language is None or (isinstance(language, str) and str(language).strip() == "")) and _extra.get("language") is not None: + language = _extra.get("language") if (session is None or (isinstance(session, str) and str(session).strip() == "")) and _extra.get("session") is not None: session = _extra.get("session") except Exception: @@ -1668,6 +1677,12 @@ async def set_session_defaults( defaults: Dict[str, Any] = {} if isinstance(collection, str) and collection.strip(): defaults["collection"] = str(collection).strip() + if isinstance(mode, str) and mode.strip(): + defaults["mode"] = str(mode).strip() + if isinstance(under, str) and under.strip(): + defaults["under"] = str(under).strip() + if isinstance(language, str) and language.strip(): + defaults["language"] = str(language).strip() # Per-connection storage (preferred) try: @@ -1959,35 +1974,61 @@ def _to_str(x, default=""): ) highlight_snippet = _to_bool(highlight_snippet, True) - # Optional mode knob: "code_first" (default for IDE), "docs_first", "balanced" - mode_str = _to_str(mode, "").strip().lower() - - # Resolve collection precedence: explicit > per-connection defaults > token defaults > env default + # Resolve collection and related hints: explicit > per-connection defaults > token defaults > env coll_hint = _to_str(collection, "").strip() + mode_hint = _to_str(mode, "").strip() + under_hint = _to_str(under, "").strip() + lang_hint = _to_str(language, "").strip() # 1) Per-connection defaults via ctx (no token required) - if (not coll_hint) and ctx is not None and getattr(ctx, "session", None) is not None: + if ctx is not None and getattr(ctx, "session", None) is not None: try: with _SESSION_CTX_LOCK: _d2 = SESSION_DEFAULTS_BY_SESSION.get(ctx.session) or {} - _sc2 = str((_d2.get("collection") or "")).strip() - if _sc2: - coll_hint = _sc2 + if not coll_hint: + _sc2 = str((_d2.get("collection") or "")).strip() + if _sc2: + coll_hint = _sc2 + if not mode_hint: + _sm2 = str((_d2.get("mode") or "")).strip() + if _sm2: + mode_hint = _sm2 + if not under_hint: + _su2 = str((_d2.get("under") or "")).strip() + if _su2: + under_hint = _su2 + if not lang_hint: + _sl2 = str((_d2.get("language") or "")).strip() + if _sl2: + lang_hint = _sl2 except Exception: pass # 2) Legacy token-based defaults - if (not coll_hint) and sid: + if sid: try: with _SESSION_LOCK: _d = SESSION_DEFAULTS.get(sid) or {} - _sc = str((_d.get("collection") or "")).strip() - if _sc: - coll_hint = _sc + if not coll_hint: + _sc = str((_d.get("collection") or "")).strip() + if _sc: + coll_hint = _sc + if not mode_hint: + _sm = str((_d.get("mode") or "")).strip() + if _sm: + mode_hint = _sm + if not under_hint: + _su = str((_d.get("under") or "")).strip() + if _su: + under_hint = _su + if not lang_hint: + _sl = str((_d.get("language") or "")).strip() + if _sl: + lang_hint = _sl except Exception: pass - # 3) Environment default + # 3) Environment default (collection only for now) env_coll = (os.environ.get("DEFAULT_COLLECTION") or os.environ.get("COLLECTION_NAME") or "").strip() if (not coll_hint) and env_coll: coll_hint = env_coll @@ -1996,6 +2037,17 @@ def _to_str(x, default=""): env_fallback = (os.environ.get("DEFAULT_COLLECTION") or os.environ.get("COLLECTION_NAME") or "my-collection").strip() collection = coll_hint or env_fallback + # Optional mode knob: "code_first" (default for IDE), "docs_first", "balanced" + if not mode: + mode = mode_hint + mode_str = _to_str(mode, "").strip().lower() + + # Apply defaults for language / under when explicit args are empty + if not language: + language = lang_hint + if not under: + under = under_hint + language = _to_str(language, "").strip() under = _to_str(under, "").strip() kind = _to_str(kind, "").strip() From f45dda5b43d33ee9f83a9aa3957811a7792de747 Mon Sep 17 00:00:00 2001 From: Reese Date: Mon, 8 Dec 2025 17:25:28 +0000 Subject: [PATCH 03/55] Enables proxying tools to a memory MCP server Adds support for proxying tools to a separate memory-based MCP server. This allows the CLI to forward specific tool calls to a memory-based context engine, enabling experimentation and local development without relying solely on the primary indexer. The `mcp-serve` command now accepts an optional `--memory-url` argument to configure the memory server. It also improves CLI argument parsing, enabling a cleaner way to specify the workspace and indexer URL. Removes the standalone indexer client as its functionality is now part of the mcp server. This configuration supports scenarios where certain tools (e.g., those beginning with "memory.") are specifically handled by the memory server, while others are directed to the main indexer. --- .github/workflows/publish-cli.yml | 36 ++++++++ ctxce-cli/package.json | 4 +- ctxce-cli/src/cli.js | 46 +++++++++-- ctxce-cli/src/indexerClient.js | 30 ------- ctxce-cli/src/mcpServer.js | 131 ++++++++++++++++++++++++++---- 5 files changed, 194 insertions(+), 53 deletions(-) create mode 100644 .github/workflows/publish-cli.yml delete mode 100644 ctxce-cli/src/indexerClient.js diff --git a/.github/workflows/publish-cli.yml b/.github/workflows/publish-cli.yml new file mode 100644 index 00000000..683e36e0 --- /dev/null +++ b/.github/workflows/publish-cli.yml @@ -0,0 +1,36 @@ +name: Publish ctxce CLI to npm + +on: + workflow_dispatch: + inputs: + version: + description: "Version to publish (ensure package.json is updated)" + required: false + push: + tags: + - "ctxce-cli-v*" + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: "20" + registry-url: "https://registry.npmjs.org" + + - name: Install dependencies + working-directory: ctxce-cli + run: npm install + + - name: Publish to npm + working-directory: ctxce-cli + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public diff --git a/ctxce-cli/package.json b/ctxce-cli/package.json index 5342dd50..f06e1189 100644 --- a/ctxce-cli/package.json +++ b/ctxce-cli/package.json @@ -1,7 +1,6 @@ { "name": "@context-engine/cli", "version": "0.0.1", - "private": true, "description": "Minimal MCP stdio proxy for Context-Engine indexer", "bin": { "ctxce": "bin/ctxce.js" @@ -14,6 +13,9 @@ "@modelcontextprotocol/sdk": "^1.24.3", "zod": "^3.25.0" }, + "publishConfig": { + "access": "public" + }, "engines": { "node": ">=18.0.0" } diff --git a/ctxce-cli/src/cli.js b/ctxce-cli/src/cli.js index 0301ea49..0af9c913 100644 --- a/ctxce-cli/src/cli.js +++ b/ctxce-cli/src/cli.js @@ -10,13 +10,45 @@ export async function runCli() { const cmd = argv[0]; if (cmd === "mcp-serve") { - // TODO: add proper argument parsing; PoC uses cwd as workspace - const workspace = process.cwd(); - const indexerUrl = process.env.CTXCE_INDEXER_URL || "http://localhost:8003/mcp"; + // Minimal flag parsing for PoC: allow passing workspace/root and indexer URL. + // Supported flags: + // --workspace / --path : workspace root (default: cwd) + // --indexer-url : override MCP indexer URL (default env CTXCE_INDEXER_URL or http://localhost:8003/mcp) + const args = argv.slice(1); + let workspace = process.cwd(); + let indexerUrl = process.env.CTXCE_INDEXER_URL || "http://localhost:8003/mcp"; + let memoryUrl = process.env.CTXCE_MEMORY_URL || null; + + for (let i = 0; i < args.length; i += 1) { + const a = args[i]; + if (a === "--workspace" || a === "--path") { + if (i + 1 < args.length) { + workspace = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--indexer-url") { + if (i + 1 < args.length) { + indexerUrl = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--memory-url") { + if (i + 1 < args.length) { + memoryUrl = args[i + 1]; + i += 1; + continue; + } + } + } // eslint-disable-next-line no-console - console.error(`[ctxce] Starting MCP bridge: workspace=${workspace}, indexerUrl=${indexerUrl}`); - await runMcpServer({ workspace, indexerUrl }); + console.error( + `[ctxce] Starting MCP bridge: workspace=${workspace}, indexerUrl=${indexerUrl}, memoryUrl=${memoryUrl || "disabled"}`, + ); + await runMcpServer({ workspace, indexerUrl, memoryUrl }); return; } @@ -26,6 +58,8 @@ export async function runCli() { const binName = "ctxce"; // eslint-disable-next-line no-console - console.error(`Usage: ${binName} mcp-serve`); + console.error( + `Usage: ${binName} mcp-serve [--workspace ] [--indexer-url ] [--memory-url ]`, + ); process.exit(1); } diff --git a/ctxce-cli/src/indexerClient.js b/ctxce-cli/src/indexerClient.js deleted file mode 100644 index 208cfe3e..00000000 --- a/ctxce-cli/src/indexerClient.js +++ /dev/null @@ -1,30 +0,0 @@ -// Minimal JSON-RPC over HTTP client for the remote Context-Engine MCP indexer - -const DEFAULT_INDEXER_URL = process.env.CTXCE_INDEXER_URL || "http://localhost:8003/mcp"; - -export class IndexerClient { - constructor(url) { - this.url = url || DEFAULT_INDEXER_URL; - } - - async call(method, params) { - const id = Date.now().toString() + Math.random().toString(16).slice(2); - const body = { jsonrpc: "2.0", id, method, params }; - - const res = await fetch(this.url, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(body), - }); - - if (!res.ok) { - throw new Error(`[indexer] HTTP ${res.status}: ${await res.text()}`); - } - - const json = await res.json(); - if (json.error) { - throw json.error; - } - return json.result; - } -} diff --git a/ctxce-cli/src/mcpServer.js b/ctxce-cli/src/mcpServer.js index d73ef6f1..3ac0a66c 100644 --- a/ctxce-cli/src/mcpServer.js +++ b/ctxce-cli/src/mcpServer.js @@ -1,3 +1,60 @@ +async function sendSessionDefaults(client, payload, label) { + if (!client) { + return; + } + try { + await client.callTool({ + name: "set_session_defaults", + arguments: payload, + }); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[ctxce] Failed to call set_session_defaults on ${label}:`, err); + } +} +function dedupeTools(tools) { + const seen = new Set(); + const out = []; + for (const tool of tools) { + const key = (tool && typeof tool.name === "string" && tool.name) || ""; + if (!key || seen.has(key)) { + if (key === "" || key !== "set_session_defaults") { + continue; + } + if (seen.has(key)) { + continue; + } + } + seen.add(key); + out.push(tool); + } + return out; +} + +async function listMemoryTools(client) { + if (!client) { + return []; + } + try { + const remote = await client.listTools(); + return Array.isArray(remote?.tools) ? remote.tools.slice() : []; + } catch (err) { + // eslint-disable-next-line no-console + console.error("[ctxce] Error calling memory tools/list:", err); + return []; + } +} + +function selectClientForTool(name, indexerClient, memoryClient) { + if (!name) { + return indexerClient; + } + const lowered = name.toLowerCase(); + if (memoryClient && (lowered.startsWith("memory.") || lowered.startsWith("mcp_memory_") || lowered.includes("memory"))) { + return memoryClient; + } + return indexerClient; +} // MCP stdio server implemented using the official MCP TypeScript SDK. // Acts as a low-level proxy for tools, forwarding tools/list and tools/call // to the remote qdrant-indexer MCP server while adding a local `ping` tool. @@ -15,6 +72,7 @@ import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprot export async function runMcpServer(options) { const workspace = options.workspace || process.cwd(); const indexerUrl = options.indexerUrl; + const memoryUrl = options.memoryUrl; const config = loadConfig(workspace); const defaultCollection = @@ -39,8 +97,8 @@ export async function runMcpServer(options) { } // High-level MCP client for the remote HTTP /mcp indexer - const clientTransport = new StreamableHTTPClientTransport(indexerUrl); - const client = new Client( + const indexerTransport = new StreamableHTTPClientTransport(indexerUrl); + const indexerClient = new Client( { name: "ctx-context-engine-bridge-http-client", version: "0.0.1", @@ -55,13 +113,40 @@ export async function runMcpServer(options) { ); try { - await client.connect(clientTransport); + await indexerClient.connect(indexerTransport); } catch (err) { // eslint-disable-next-line no-console console.error("[ctxce] Failed to connect MCP HTTP client to indexer:", err); throw err; } + let memoryClient = null; + if (memoryUrl) { + try { + const memoryTransport = new StreamableHTTPClientTransport(memoryUrl); + memoryClient = new Client( + { + name: "ctx-context-engine-bridge-memory-client", + version: "0.0.1", + }, + { + capabilities: { + tools: {}, + resources: {}, + prompts: {}, + }, + }, + ); + await memoryClient.connect(memoryTransport); + // eslint-disable-next-line no-console + console.error("[ctxce] Connected memory MCP client:", memoryUrl); + } catch (err) { + // eslint-disable-next-line no-console + console.error("[ctxce] Failed to connect memory MCP client:", err); + memoryClient = null; + } + } + // Derive a simple session identifier for this bridge process. In the // future this can be made user-aware (e.g. from auth), but for now we // keep it deterministic per workspace to help the indexer reuse @@ -83,14 +168,9 @@ export async function runMcpServer(options) { } if (Object.keys(defaultsPayload).length > 1) { - try { - await client.callTool({ - name: "set_session_defaults", - arguments: defaultsPayload, - }); - } catch (err) { - // eslint-disable-next-line no-console - console.error("[ctxce] Failed to call set_session_defaults on indexer:", err); + await sendSessionDefaults(indexerClient, defaultsPayload, "indexer"); + if (memoryClient) { + await sendSessionDefaults(memoryClient, defaultsPayload, "memory"); } } @@ -110,7 +190,7 @@ export async function runMcpServer(options) { server.setRequestHandler(ListToolsRequestSchema, async () => { let remote; try { - remote = await client.listTools(); + remote = await indexerClient.listTools(); } catch (err) { // eslint-disable-next-line no-console console.error("[ctxce] Error calling remote tools/list:", err); @@ -120,8 +200,9 @@ export async function runMcpServer(options) { // eslint-disable-next-line no-console console.error("[ctxce] tools/list remote result:", JSON.stringify(remote)); - const tools = Array.isArray(remote?.tools) ? remote.tools.slice() : []; - tools.push(buildPingTool()); + const indexerTools = Array.isArray(remote?.tools) ? remote.tools.slice() : []; + const memoryTools = await listMemoryTools(memoryClient); + const tools = dedupeTools([...indexerTools, ...memoryTools, buildPingTool()]); return { tools }; }); @@ -145,7 +226,7 @@ export async function runMcpServer(options) { }; } - // Attach session id so the indexer can apply per-session defaults. + // Attach session id so the target server can apply per-session defaults. if (sessionId && (args === undefined || args === null || typeof args === "object")) { const obj = args && typeof args === "object" ? { ...args } : {}; if (!Object.prototype.hasOwnProperty.call(obj, "session")) { @@ -154,7 +235,25 @@ export async function runMcpServer(options) { args = obj; } - const result = await client.callTool({ + if (name === "set_session_defaults") { + const indexerResult = await indexerClient.callTool({ name, arguments: args }); + if (memoryClient) { + try { + await memoryClient.callTool({ name, arguments: args }); + } catch (err) { + // eslint-disable-next-line no-console + console.error("[ctxce] Memory set_session_defaults failed:", err); + } + } + return indexerResult; + } + + const targetClient = selectClientForTool(name, indexerClient, memoryClient); + if (!targetClient) { + throw new Error(`Tool ${name} not available on any configured MCP server`); + } + + const result = await targetClient.callTool({ name, arguments: args, }); From 70b6257d24a3a983b4244d4d24c06c1259e4f2c2 Mon Sep 17 00:00:00 2001 From: Reese Date: Mon, 8 Dec 2025 17:52:21 +0000 Subject: [PATCH 04/55] ctxce cli - rename to mcp bridge --- {ctxce-cli => ctx-mcp-bridge}/bin/ctxce.js | 0 {ctxce-cli => ctx-mcp-bridge}/package-lock.json | 0 {ctxce-cli => ctx-mcp-bridge}/package.json | 7 ++++--- {ctxce-cli => ctx-mcp-bridge}/src/cli.js | 0 {ctxce-cli => ctx-mcp-bridge}/src/mcpServer.js | 0 5 files changed, 4 insertions(+), 3 deletions(-) rename {ctxce-cli => ctx-mcp-bridge}/bin/ctxce.js (100%) rename {ctxce-cli => ctx-mcp-bridge}/package-lock.json (100%) rename {ctxce-cli => ctx-mcp-bridge}/package.json (59%) rename {ctxce-cli => ctx-mcp-bridge}/src/cli.js (100%) rename {ctxce-cli => ctx-mcp-bridge}/src/mcpServer.js (100%) diff --git a/ctxce-cli/bin/ctxce.js b/ctx-mcp-bridge/bin/ctxce.js similarity index 100% rename from ctxce-cli/bin/ctxce.js rename to ctx-mcp-bridge/bin/ctxce.js diff --git a/ctxce-cli/package-lock.json b/ctx-mcp-bridge/package-lock.json similarity index 100% rename from ctxce-cli/package-lock.json rename to ctx-mcp-bridge/package-lock.json diff --git a/ctxce-cli/package.json b/ctx-mcp-bridge/package.json similarity index 59% rename from ctxce-cli/package.json rename to ctx-mcp-bridge/package.json index f06e1189..fcc2e409 100644 --- a/ctxce-cli/package.json +++ b/ctx-mcp-bridge/package.json @@ -1,9 +1,10 @@ { - "name": "@context-engine/cli", + "name": "@context-engine/mcp-bridge", "version": "0.0.1", - "description": "Minimal MCP stdio proxy for Context-Engine indexer", + "description": "Context Engine MCP bridge (stdio proxy combining indexer + memory servers)", "bin": { - "ctxce": "bin/ctxce.js" + "ctxce": "bin/ctxce.js", + "ctxce-bridge": "bin/ctxce.js" }, "type": "module", "scripts": { diff --git a/ctxce-cli/src/cli.js b/ctx-mcp-bridge/src/cli.js similarity index 100% rename from ctxce-cli/src/cli.js rename to ctx-mcp-bridge/src/cli.js diff --git a/ctxce-cli/src/mcpServer.js b/ctx-mcp-bridge/src/mcpServer.js similarity index 100% rename from ctxce-cli/src/mcpServer.js rename to ctx-mcp-bridge/src/mcpServer.js From 5242b069a8df39800f547f1ada9d921aad780bab Mon Sep 17 00:00:00 2001 From: Reese Date: Mon, 8 Dec 2025 18:13:27 +0000 Subject: [PATCH 05/55] Update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 1e578035..f5bdce85 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ # Model artifacts models/ +# Node +node_modules/ + # Qdrant snapshots qdrant/snapshots/ From 02fafd901251e42f99fc1ddc2ddbc6ff99517550 Mon Sep 17 00:00:00 2001 From: Reese Date: Mon, 8 Dec 2025 20:54:53 +0000 Subject: [PATCH 06/55] Renames package and adds publish script Renames the package to align with the new naming convention and avoid conflicts. Adds a publish script to simplify the release process, including authentication and version bumping. --- ctx-mcp-bridge/.gitignore | 1 + ctx-mcp-bridge/package-lock.json | 7 ++++--- ctx-mcp-bridge/package.json | 2 +- ctx-mcp-bridge/publish.sh | 34 ++++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 ctx-mcp-bridge/.gitignore create mode 100644 ctx-mcp-bridge/publish.sh diff --git a/ctx-mcp-bridge/.gitignore b/ctx-mcp-bridge/.gitignore new file mode 100644 index 00000000..9e30eb9b --- /dev/null +++ b/ctx-mcp-bridge/.gitignore @@ -0,0 +1 @@ +*.tgz \ No newline at end of file diff --git a/ctx-mcp-bridge/package-lock.json b/ctx-mcp-bridge/package-lock.json index 16b6f8a1..045974de 100644 --- a/ctx-mcp-bridge/package-lock.json +++ b/ctx-mcp-bridge/package-lock.json @@ -1,18 +1,19 @@ { - "name": "@context-engine/cli", + "name": "@context-engine/mcp-bridge", "version": "0.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@context-engine/cli", + "name": "@context-engine/mcp-bridge", "version": "0.0.1", "dependencies": { "@modelcontextprotocol/sdk": "^1.24.3", "zod": "^3.25.0" }, "bin": { - "ctxce": "bin/ctxce.js" + "ctxce": "bin/ctxce.js", + "ctxce-bridge": "bin/ctxce.js" }, "engines": { "node": ">=18.0.0" diff --git a/ctx-mcp-bridge/package.json b/ctx-mcp-bridge/package.json index fcc2e409..68985fe2 100644 --- a/ctx-mcp-bridge/package.json +++ b/ctx-mcp-bridge/package.json @@ -1,5 +1,5 @@ { - "name": "@context-engine/mcp-bridge", + "name": "@context-engine-bridge/context-engine-mcp-bridge", "version": "0.0.1", "description": "Context Engine MCP bridge (stdio proxy combining indexer + memory servers)", "bin": { diff --git a/ctx-mcp-bridge/publish.sh b/ctx-mcp-bridge/publish.sh new file mode 100644 index 00000000..7d86e455 --- /dev/null +++ b/ctx-mcp-bridge/publish.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Simple helper to login (if needed) and publish the package. +# Usage: +# ./publish.sh # publishes current version +# ./publish.sh 0.0.2 # bumps version to 0.0.2 then publishes + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +PACKAGE_NAME="@context-engine-bridge/context-engine-mcp-bridge" + +echo "[publish] Verifying npm authentication..." +if ! npm whoami >/dev/null 2>&1; then + echo "[publish] Not logged in; running npm login" + npm login +else + echo "[publish] Already authenticated as $(npm whoami)" +fi + +if [[ $# -gt 0 ]]; then + VERSION="$1" + echo "[publish] Bumping version to $VERSION" + npm version "$VERSION" --no-git-tag-version +fi + +echo "[publish] Packing $PACKAGE_NAME for verification..." +npm pack >/dev/null + +echo "[publish] Publishing $PACKAGE_NAME..." +npm publish --access public + +echo "[publish] Done!" From 3cd07f8940927b1e218fd9ad62a3ef0d536003e7 Mon Sep 17 00:00:00 2001 From: Reese Date: Mon, 8 Dec 2025 20:59:54 +0000 Subject: [PATCH 07/55] mcp bridge: Updates publish workflow to new CLI package name Updates the publish workflow to reflect the renaming of the CLI package. This ensures that the CLI is published correctly from the correct directory. Also adds `--provenance` flag to `npm publish`. --- .github/workflows/publish-cli.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-cli.yml b/.github/workflows/publish-cli.yml index 683e36e0..01f1aebb 100644 --- a/.github/workflows/publish-cli.yml +++ b/.github/workflows/publish-cli.yml @@ -26,11 +26,9 @@ jobs: registry-url: "https://registry.npmjs.org" - name: Install dependencies - working-directory: ctxce-cli + working-directory: ctx-mcp-bridge run: npm install - name: Publish to npm - working-directory: ctxce-cli - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: npm publish --access public + working-directory: ctx-mcp-bridge + run: npm publish --access public --provenance From 395d049cf279b7e56f06f998ef63de71110ffdc6 Mon Sep 17 00:00:00 2001 From: Reese Date: Mon, 8 Dec 2025 23:34:30 +0000 Subject: [PATCH 08/55] fix: harden MCP bridge tools/list handling on Windows - add timeouts around indexer and memory listTools calls - remove local ping tool and simplify tools/call proxying - add optional debug logging via CTXCE_DEBUG_LOG - avoid exceptions in tools/list logging that could hang refresh Also upgrades the package version. --- ctx-mcp-bridge/docs/debugging.md | 20 +++++ ctx-mcp-bridge/package-lock.json | 4 +- ctx-mcp-bridge/package.json | 2 +- ctx-mcp-bridge/src/mcpServer.js | 133 +++++++++++++++++++------------ 4 files changed, 104 insertions(+), 55 deletions(-) create mode 100644 ctx-mcp-bridge/docs/debugging.md diff --git a/ctx-mcp-bridge/docs/debugging.md b/ctx-mcp-bridge/docs/debugging.md new file mode 100644 index 00000000..e5904743 --- /dev/null +++ b/ctx-mcp-bridge/docs/debugging.md @@ -0,0 +1,20 @@ +{ + "mcpServers": { + "context-engine": { + "command": "node", + "args": [ + "C:/Users/Admin/Documents/GitHub/Context-Engine/ctx-mcp-bridge/bin/ctxce.js", + "mcp-serve", + "--indexer-url", + "http://192.168.100.249:30806/mcp", + "--memory-url", + "http://192.168.100.249:30804/mcp", + "--workspace", + "C:/Users/Admin/Documents/GitHub/Pirate Survivors" + ], + "env": { + "CTXCE_DEBUG_LOG": "C:/Users/Admin/ctxce-mcp.log" + } + } + } +} \ No newline at end of file diff --git a/ctx-mcp-bridge/package-lock.json b/ctx-mcp-bridge/package-lock.json index 045974de..04b28ef3 100644 --- a/ctx-mcp-bridge/package-lock.json +++ b/ctx-mcp-bridge/package-lock.json @@ -1,11 +1,11 @@ { - "name": "@context-engine/mcp-bridge", + "name": "@context-engine-bridge/context-engine-mcp-bridge", "version": "0.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@context-engine/mcp-bridge", + "name": "@context-engine-bridge/context-engine-mcp-bridge", "version": "0.0.1", "dependencies": { "@modelcontextprotocol/sdk": "^1.24.3", diff --git a/ctx-mcp-bridge/package.json b/ctx-mcp-bridge/package.json index 68985fe2..0483505e 100644 --- a/ctx-mcp-bridge/package.json +++ b/ctx-mcp-bridge/package.json @@ -1,6 +1,6 @@ { "name": "@context-engine-bridge/context-engine-mcp-bridge", - "version": "0.0.1", + "version": "0.0.2", "description": "Context Engine MCP bridge (stdio proxy combining indexer + memory servers)", "bin": { "ctxce": "bin/ctxce.js", diff --git a/ctx-mcp-bridge/src/mcpServer.js b/ctx-mcp-bridge/src/mcpServer.js index 3ac0a66c..7b997e07 100644 --- a/ctx-mcp-bridge/src/mcpServer.js +++ b/ctx-mcp-bridge/src/mcpServer.js @@ -1,3 +1,15 @@ +function debugLog(message) { + try { + const text = typeof message === "string" ? message : String(message); + console.error(text); + const dest = process.env.CTXCE_DEBUG_LOG; + if (dest) { + fs.appendFileSync(dest, `${new Date().toISOString()} ${text}\n`, "utf8"); + } + } catch { + } +} + async function sendSessionDefaults(client, payload, label) { if (!client) { return; @@ -36,15 +48,52 @@ async function listMemoryTools(client) { return []; } try { - const remote = await client.listTools(); + const remote = await withTimeout( + client.listTools(), + 5000, + "memory tools/list", + ); return Array.isArray(remote?.tools) ? remote.tools.slice() : []; } catch (err) { - // eslint-disable-next-line no-console - console.error("[ctxce] Error calling memory tools/list:", err); + debugLog("[ctxce] Error calling memory tools/list: " + String(err)); return []; } } +function withTimeout(promise, ms, label) { + return new Promise((resolve, reject) => { + let settled = false; + const timer = setTimeout(() => { + if (settled) { + return; + } + settled = true; + const errorMessage = + label != null + ? `[ctxce] Timeout after ${ms}ms in ${label}` + : `[ctxce] Timeout after ${ms}ms`; + reject(new Error(errorMessage)); + }, ms); + promise + .then((value) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + resolve(value); + }) + .catch((err) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + reject(err); + }); + }); +} + function selectClientForTool(name, indexerClient, memoryClient) { if (!name) { return indexerClient; @@ -84,8 +133,7 @@ export async function runMcpServer(options) { const defaultUnder = config && typeof config.default_under === "string" ? config.default_under : null; - // eslint-disable-next-line no-console - console.error( + debugLog( `[ctxce] MCP low-level stdio bridge starting: workspace=${workspace}, indexerUrl=${indexerUrl}`, ); @@ -115,8 +163,7 @@ export async function runMcpServer(options) { try { await indexerClient.connect(indexerTransport); } catch (err) { - // eslint-disable-next-line no-console - console.error("[ctxce] Failed to connect MCP HTTP client to indexer:", err); + debugLog("[ctxce] Failed to connect MCP HTTP client to indexer: " + String(err)); throw err; } @@ -138,11 +185,9 @@ export async function runMcpServer(options) { }, ); await memoryClient.connect(memoryTransport); - // eslint-disable-next-line no-console - console.error("[ctxce] Connected memory MCP client:", memoryUrl); + debugLog(`[ctxce] Connected memory MCP client: ${memoryUrl}`); } catch (err) { - // eslint-disable-next-line no-console - console.error("[ctxce] Failed to connect memory MCP client:", err); + debugLog("[ctxce] Failed to connect memory MCP client: " + String(err)); memoryClient = null; } } @@ -186,45 +231,47 @@ export async function runMcpServer(options) { }, ); - // tools/list → fetch tools from remote indexer and append local ping tool + // tools/list → fetch tools from remote indexer server.setRequestHandler(ListToolsRequestSchema, async () => { let remote; try { - remote = await indexerClient.listTools(); + debugLog("[ctxce] tools/list: fetching tools from indexer"); + remote = await withTimeout( + indexerClient.listTools(), + 10000, + "indexer tools/list", + ); } catch (err) { - // eslint-disable-next-line no-console - console.error("[ctxce] Error calling remote tools/list:", err); - return { tools: [buildPingTool()] }; + debugLog("[ctxce] Error calling remote tools/list: " + String(err)); + const memoryToolsFallback = await listMemoryTools(memoryClient); + const toolsFallback = dedupeTools([...memoryToolsFallback]); + return { tools: toolsFallback }; } - // eslint-disable-next-line no-console - console.error("[ctxce] tools/list remote result:", JSON.stringify(remote)); + try { + const toolNames = + remote && Array.isArray(remote.tools) + ? remote.tools.map((t) => (t && typeof t.name === "string" ? t.name : "")) + : []; + debugLog("[ctxce] tools/list remote result tools: " + JSON.stringify(toolNames)); + } catch (err) { + debugLog("[ctxce] tools/list remote result: " + String(err)); + } const indexerTools = Array.isArray(remote?.tools) ? remote.tools.slice() : []; const memoryTools = await listMemoryTools(memoryClient); - const tools = dedupeTools([...indexerTools, ...memoryTools, buildPingTool()]); + const tools = dedupeTools([...indexerTools, ...memoryTools]); + debugLog(`[ctxce] tools/list: returning ${tools.length} tools`); return { tools }; }); - // tools/call → handle ping locally, everything else is proxied to indexer + // tools/call → proxied to indexer or memory server server.setRequestHandler(CallToolRequestSchema, async (request) => { const params = request.params || {}; const name = params.name; let args = params.arguments; - if (name === "ping") { - const branch = detectGitBranch(workspace); - const text = args && typeof args.text === "string" ? args.text : "pong"; - const suffix = branch ? ` (branch=${branch})` : ""; - return { - content: [ - { - type: "text", - text: `${text}${suffix}`, - }, - ], - }; - } + debugLog(`[ctxce] tools/call: ${name || ""}`); // Attach session id so the target server can apply per-session defaults. if (sessionId && (args === undefined || args === null || typeof args === "object")) { @@ -241,8 +288,7 @@ export async function runMcpServer(options) { try { await memoryClient.callTool({ name, arguments: args }); } catch (err) { - // eslint-disable-next-line no-console - console.error("[ctxce] Memory set_session_defaults failed:", err); + debugLog("[ctxce] Memory set_session_defaults failed: " + String(err)); } } return indexerResult; @@ -295,23 +341,6 @@ function loadConfig(startDir) { return null; } -function buildPingTool() { - return { - name: "ping", - description: "Basic ping tool exposed by the ctx bridge", - inputSchema: { - type: "object", - properties: { - text: { - type: "string", - description: "Optional text to echo back.", - }, - }, - required: [], - }, - }; -} - function detectGitBranch(workspace) { try { const out = execSync("git rev-parse --abbrev-ref HEAD", { From 896cd3486c25284cc1930c895982b4c8b1c84f15 Mon Sep 17 00:00:00 2001 From: Reese Date: Tue, 9 Dec 2025 01:17:23 +0000 Subject: [PATCH 09/55] bridge: chmod publish script --- ctx-mcp-bridge/publish.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ctx-mcp-bridge/publish.sh diff --git a/ctx-mcp-bridge/publish.sh b/ctx-mcp-bridge/publish.sh old mode 100644 new mode 100755 From 10f9a2b5ce336bd9790e736be80b6f49a8f62d2f Mon Sep 17 00:00:00 2001 From: Reese Date: Tue, 9 Dec 2025 04:18:25 +0000 Subject: [PATCH 10/55] mcp bridge poc: extension mcp config wiring (stdio) --- .../context-engine-uploader/extension.js | 93 +++++++++++++++++-- .../context-engine-uploader/package.json | 10 +- 2 files changed, 95 insertions(+), 8 deletions(-) diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index a72b7cdc..0cd57765 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -1022,6 +1022,70 @@ function buildChildEnv(options) { } return env; } +function normalizeBridgeUrl(url) { + if (!url || typeof url !== 'string') { + return ''; + } + const trimmed = url.trim(); + if (!trimmed) { + return ''; + } + try { + const parsed = new URL(trimmed); + if (parsed.pathname.endsWith('/sse')) { + parsed.pathname = parsed.pathname.replace(/\/sse$/, '/mcp'); + return parsed.toString(); + } + } catch (_) { + // fall through to return trimmed + } + return trimmed; +} + +function normalizeWorkspaceForBridge(workspacePath) { + if (!workspacePath || typeof workspacePath !== 'string') { + return ''; + } + try { + const resolved = path.resolve(workspacePath); + if (process.platform === 'win32') { + return resolved.replace(/\//g, '\\'); + } + return resolved; + } catch (_) { + return workspacePath; + } +} + +function buildBridgeServerConfig(workspacePath, indexerUrl, memoryUrl) { + const isWindows = process.platform === 'win32'; + const args = [ + '@context-engine-bridge/context-engine-mcp-bridge', + 'mcp-serve' + ]; + if (workspacePath) { + args.push('--workspace', normalizeWorkspaceForBridge(workspacePath)); + } + if (indexerUrl) { + args.push('--indexer-url', indexerUrl); + } + if (memoryUrl) { + args.push('--memory-url', memoryUrl); + } + if (isWindows) { + return { + command: 'cmd', + args: ['/c', 'npx', ...args], + env: {} + }; + } + return { + command: 'npx', + args, + env: {} + }; +} + async function writeMcpConfig() { const settings = vscode.workspace.getConfiguration('contextEngineUploader'); const claudeEnabled = settings.get('mcpClaudeEnabled', true); @@ -1034,9 +1098,15 @@ async function writeMcpConfig() { } const transportModeRaw = (settings.get('mcpTransportMode') || 'sse-remote'); const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; + const serverModeRaw = (settings.get('mcpServerMode') || 'bridge'); + const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; let indexerUrl = (settings.get('mcpIndexerUrl') || 'http://localhost:8001/sse').trim(); let memoryUrl = (settings.get('mcpMemoryUrl') || 'http://localhost:8000/sse').trim(); + if (serverMode === 'bridge') { + indexerUrl = normalizeBridgeUrl(indexerUrl); + memoryUrl = normalizeBridgeUrl(memoryUrl); + } let wroteAny = false; let hookWrote = false; if (claudeEnabled) { @@ -1044,14 +1114,15 @@ async function writeMcpConfig() { if (!root) { vscode.window.showErrorMessage('Context Engine Uploader: open a folder before writing .mcp.json.'); } else { - const result = await writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode); + const result = await writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode, serverMode); wroteAny = wroteAny || result; } } if (windsurfEnabled) { const customPath = (settings.get('windsurfMcpPath') || '').trim(); const windsPath = customPath || getDefaultWindsurfMcpPath(); - const result = await writeWindsurfMcpServers(windsPath, indexerUrl, memoryUrl, transportMode); + const workspaceHint = getWorkspaceFolderPath(); + const result = await writeWindsurfMcpServers(windsPath, indexerUrl, memoryUrl, transportMode, serverMode, workspaceHint); wroteAny = wroteAny || result; } if (claudeHookEnabled) { @@ -1498,7 +1569,7 @@ function getDefaultWindsurfMcpPath() { return path.join(os.homedir(), '.codeium', 'windsurf', 'mcp_config.json'); } -async function writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode) { +async function writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode, serverMode = 'bridge') { const configPath = path.join(root, '.mcp.json'); let config = { mcpServers: {} }; if (fs.existsSync(configPath)) { @@ -1521,7 +1592,12 @@ async function writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode) const servers = config.mcpServers; const mode = (typeof transportMode === 'string' ? transportMode.trim() : 'sse-remote') || 'sse-remote'; - if (mode === 'http') { + if (serverMode === 'bridge') { + const bridgeServer = buildBridgeServerConfig(root, indexerUrl, memoryUrl); + servers['context-engine'] = bridgeServer; + delete servers['qdrant-indexer']; + delete servers.memory; + } else if (mode === 'http') { // Direct HTTP MCP endpoints for Claude (.mcp.json) if (indexerUrl) { servers['qdrant-indexer'] = { @@ -1572,7 +1648,7 @@ async function writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode) } } -async function writeWindsurfMcpServers(configPath, indexerUrl, memoryUrl, transportMode) { +async function writeWindsurfMcpServers(configPath, indexerUrl, memoryUrl, transportMode, serverMode = 'bridge', workspaceHint) { try { fs.mkdirSync(path.dirname(configPath), { recursive: true }); } catch (error) { @@ -1601,7 +1677,12 @@ async function writeWindsurfMcpServers(configPath, indexerUrl, memoryUrl, transp const servers = config.mcpServers; const mode = (typeof transportMode === 'string' ? transportMode.trim() : 'sse-remote') || 'sse-remote'; - if (mode === 'http') { + if (serverMode === 'bridge') { + const bridgeServer = buildBridgeServerConfig(workspaceHint || '', indexerUrl, memoryUrl); + servers['context-engine'] = bridgeServer; + delete servers['qdrant-indexer']; + delete servers.memory; + } else if (mode === 'http') { // Direct HTTP MCP endpoints for Windsurf mcp_config.json if (indexerUrl) { servers['qdrant-indexer'] = { diff --git a/vscode-extension/context-engine-uploader/package.json b/vscode-extension/context-engine-uploader/package.json index 9d6c06ba..b1269b94 100644 --- a/vscode-extension/context-engine-uploader/package.json +++ b/vscode-extension/context-engine-uploader/package.json @@ -166,15 +166,21 @@ "default": "http", "description": "Transport mode for Claude/Windsurf MCP configs: SSE via mcp-remote (sse-remote) or direct HTTP /mcp endpoints (http)." }, + "contextEngineUploader.mcpServerMode": { + "type": "string", + "enum": ["bridge", "direct"], + "default": "bridge", + "description": "Preferred MCP server wiring. \"bridge\" uses the @context-engine-bridge/context-engine-mcp-bridge CLI (recommended). \"direct\" writes separate Qdrant indexer and memory MCP entries like previous versions." + }, "contextEngineUploader.mcpIndexerUrl": { "type": "string", "default": "http://localhost:8001/sse", - "description": "Claude Code MCP server URL for the Qdrant indexer. Used when writing the project-local .mcp.json via 'Write MCP Config'." + "description": "Claude Code MCP server URL for the Qdrant indexer. Used when writing MCP configs." }, "contextEngineUploader.mcpMemoryUrl": { "type": "string", "default": "http://localhost:8000/sse", - "description": "Claude Code MCP server URL for the memory/search MCP server. Used when writing the project-local .mcp.json via 'Write MCP Config'." + "description": "Claude Code MCP server URL for the memory/search MCP server. Used when writing MCP configs." }, "contextEngineUploader.ctxIndexerUrl": { "type": "string", From 89ba27e493c5c2b2cab17a74edb456258a42db25 Mon Sep 17 00:00:00 2001 From: Reese Date: Tue, 9 Dec 2025 06:42:55 +0000 Subject: [PATCH 11/55] Add HTTP MCP bridge and IDE auto-wiring - Extend ctx-mcp-bridge CLI with mcp-http-serve for HTTP-based MCP - Share core bridge setup between stdio and HTTP transports - Start ctxce HTTP bridge from the VS Code extension with workspace context - Wire Claude/Windsurf MCP configs to bridge HTTP URL based on settings - Keep existing stdio bridge behavior selectable via server/transport modes --- ctx-mcp-bridge/package.json | 4 +- ctx-mcp-bridge/src/cli.js | 54 +++- ctx-mcp-bridge/src/mcpServer.js | 111 +++++++- .../context-engine-uploader/extension.js | 265 +++++++++++++++++- .../context-engine-uploader/package.json | 19 +- 5 files changed, 437 insertions(+), 16 deletions(-) diff --git a/ctx-mcp-bridge/package.json b/ctx-mcp-bridge/package.json index 0483505e..cddce66f 100644 --- a/ctx-mcp-bridge/package.json +++ b/ctx-mcp-bridge/package.json @@ -1,7 +1,7 @@ { "name": "@context-engine-bridge/context-engine-mcp-bridge", - "version": "0.0.2", - "description": "Context Engine MCP bridge (stdio proxy combining indexer + memory servers)", + "version": "0.0.3", + "description": "Context Engine MCP bridge (http/stdio proxy combining indexer + memory servers)", "bin": { "ctxce": "bin/ctxce.js", "ctxce-bridge": "bin/ctxce.js" diff --git a/ctx-mcp-bridge/src/cli.js b/ctx-mcp-bridge/src/cli.js index 0af9c913..3df486d8 100644 --- a/ctx-mcp-bridge/src/cli.js +++ b/ctx-mcp-bridge/src/cli.js @@ -3,12 +3,62 @@ import process from "node:process"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { runMcpServer } from "./mcpServer.js"; +import { runMcpServer, runHttpMcpServer } from "./mcpServer.js"; export async function runCli() { const argv = process.argv.slice(2); const cmd = argv[0]; + if (cmd === "mcp-http-serve") { + const args = argv.slice(1); + let workspace = process.cwd(); + let indexerUrl = process.env.CTXCE_INDEXER_URL || "http://localhost:8003/mcp"; + let memoryUrl = process.env.CTXCE_MEMORY_URL || null; + let port = Number.parseInt(process.env.CTXCE_HTTP_PORT || "30810", 10) || 30810; + + for (let i = 0; i < args.length; i += 1) { + const a = args[i]; + if (a === "--workspace" || a === "--path") { + if (i + 1 < args.length) { + workspace = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--indexer-url") { + if (i + 1 < args.length) { + indexerUrl = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--memory-url") { + if (i + 1 < args.length) { + memoryUrl = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--port") { + if (i + 1 < args.length) { + const parsed = Number.parseInt(args[i + 1], 10); + if (!Number.isNaN(parsed) && parsed > 0) { + port = parsed; + } + i += 1; + continue; + } + } + } + + // eslint-disable-next-line no-console + console.error( + `[ctxce] Starting HTTP MCP bridge: workspace=${workspace}, port=${port}, indexerUrl=${indexerUrl}, memoryUrl=${memoryUrl || "disabled"}`, + ); + await runHttpMcpServer({ workspace, indexerUrl, memoryUrl, port }); + return; + } + if (cmd === "mcp-serve") { // Minimal flag parsing for PoC: allow passing workspace/root and indexer URL. // Supported flags: @@ -59,7 +109,7 @@ export async function runCli() { // eslint-disable-next-line no-console console.error( - `Usage: ${binName} mcp-serve [--workspace ] [--indexer-url ] [--memory-url ]`, + `Usage: ${binName} mcp-serve [--workspace ] [--indexer-url ] [--memory-url ] | ${binName} mcp-http-serve [--workspace ] [--indexer-url ] [--memory-url ] [--port ]`, ); process.exit(1); } diff --git a/ctx-mcp-bridge/src/mcpServer.js b/ctx-mcp-bridge/src/mcpServer.js index 7b997e07..5da6d5d7 100644 --- a/ctx-mcp-bridge/src/mcpServer.js +++ b/ctx-mcp-bridge/src/mcpServer.js @@ -112,13 +112,15 @@ import process from "node:process"; import fs from "node:fs"; import path from "node:path"; import { execSync } from "node:child_process"; +import { createServer } from "node:http"; import { Server } from "@modelcontextprotocol/sdk/server/index.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js"; -export async function runMcpServer(options) { +async function createBridgeServer(options) { const workspace = options.workspace || process.cwd(); const indexerUrl = options.indexerUrl; const memoryUrl = options.memoryUrl; @@ -164,7 +166,6 @@ export async function runMcpServer(options) { await indexerClient.connect(indexerTransport); } catch (err) { debugLog("[ctxce] Failed to connect MCP HTTP client to indexer: " + String(err)); - throw err; } let memoryClient = null; @@ -306,10 +307,116 @@ export async function runMcpServer(options) { return result; }); + return server; +} + +export async function runMcpServer(options) { + const server = await createBridgeServer(options); const transport = new StdioServerTransport(); await server.connect(transport); } +export async function runHttpMcpServer(options) { + const server = await createBridgeServer(options); + const port = + typeof options.port === "number" + ? options.port + : Number.parseInt(process.env.CTXCE_HTTP_PORT || "30810", 10) || 30810; + + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined, + }); + + await server.connect(transport); + + const httpServer = createServer((req, res) => { + try { + if (!req.url || !req.url.startsWith("/mcp")) { + res.statusCode = 404; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32000, message: "Not found" }, + id: null, + }), + ); + return; + } + + if (req.method !== "POST") { + res.statusCode = 405; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32000, message: "Method not allowed" }, + id: null, + }), + ); + return; + } + + let body = ""; + req.on("data", (chunk) => { + body += chunk; + }); + req.on("end", async () => { + let parsed; + try { + parsed = body ? JSON.parse(body) : {}; + } catch (err) { + debugLog("[ctxce] Failed to parse HTTP MCP request body: " + String(err)); + res.statusCode = 400; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32700, message: "Invalid JSON" }, + id: null, + }), + ); + return; + } + + try { + await transport.handleRequest(req, res, parsed); + } catch (err) { + debugLog("[ctxce] Error handling HTTP MCP request: " + String(err)); + if (!res.headersSent) { + res.statusCode = 500; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32603, message: "Internal server error" }, + id: null, + }), + ); + } + } + }); + } catch (err) { + debugLog("[ctxce] Unexpected error in HTTP MCP server: " + String(err)); + if (!res.headersSent) { + res.statusCode = 500; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32603, message: "Internal server error" }, + id: null, + }), + ); + } + } + }); + + httpServer.listen(port, () => { + debugLog(`[ctxce] HTTP MCP bridge listening on port ${port}`); + }); +} + function loadConfig(startDir) { try { let dir = startDir; diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index 0cd57765..6615a98a 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -17,6 +17,9 @@ let watchedTargetPath; let indexedWatchDisposables = []; let globalStoragePath; let pythonOverridePath; +let httpBridgeProcess; +let httpBridgePort; +let httpBridgeWorkspace; const REQUIRED_PYTHON_MODULES = ['requests', 'urllib3', 'charset_normalizer']; const DEFAULT_CONTAINER_ROOT = '/work'; // const CLAUDE_HOOK_COMMAND = '/home/coder/project/Context-Engine/ctx-hook-simple.sh'; @@ -83,6 +86,17 @@ function activate(context) { const showLogsDisposable = vscode.commands.registerCommand('contextEngineUploader.showUploadServiceLogs', () => { try { openUploadServiceLogsTerminal(); } catch (e) { log(`Show logs failed: ${e && e.message ? e.message : String(e)}`); } }); + const startBridgeDisposable = vscode.commands.registerCommand('contextEngineUploader.startMcpHttpBridge', () => { + startHttpBridgeProcess().catch(error => { + log(`HTTP MCP bridge start failed: ${error instanceof Error ? error.message : String(error)}`); + vscode.window.showErrorMessage('Context Engine Uploader: failed to start HTTP MCP bridge. Check Output for details.'); + }); + }); + const stopBridgeDisposable = vscode.commands.registerCommand('contextEngineUploader.stopMcpHttpBridge', () => { + stopHttpBridgeProcess().catch(error => { + log(`HTTP MCP bridge stop failed: ${error instanceof Error ? error.message : String(error)}`); + }); + }); const promptEnhanceDisposable = vscode.commands.registerCommand('contextEngineUploader.promptEnhance', () => { enhanceSelectionWithUnicorn().catch(error => { log(`Prompt+ failed: ${error instanceof Error ? error.message : String(error)}`); @@ -109,6 +123,16 @@ function activate(context) { // Best-effort auto-update of MCP + hook configurations when settings change writeMcpConfig().catch(error => log(`Auto MCP config write failed: ${error instanceof Error ? error.message : String(error)}`)); } + if ( + event.affectsConfiguration('contextEngineUploader.autoStartMcpBridge') || + event.affectsConfiguration('contextEngineUploader.mcpBridgePort') || + event.affectsConfiguration('contextEngineUploader.mcpIndexerUrl') || + event.affectsConfiguration('contextEngineUploader.mcpMemoryUrl') || + event.affectsConfiguration('contextEngineUploader.mcpServerMode') || + event.affectsConfiguration('contextEngineUploader.mcpTransportMode') + ) { + handleHttpBridgeSettingsChanged().catch(error => log(`HTTP MCP bridge restart failed: ${error instanceof Error ? error.message : String(error)}`)); + } }); const workspaceDisposable = vscode.workspace.onDidChangeWorkspaceFolders(() => { ensureTargetPathConfigured(); @@ -127,6 +151,8 @@ function activate(context) { uploadGitHistoryDisposable, showLogsDisposable, promptEnhanceDisposable, + startBridgeDisposable, + stopBridgeDisposable, mcpConfigDisposable, ctxConfigDisposable, configDisposable, @@ -146,6 +172,9 @@ function activate(context) { // Legacy behavior: scaffold ctx_config.json/.env directly when MCP auto-write is disabled writeCtxConfig().catch(error => log(`CTX config auto-scaffold on activation failed: ${error instanceof Error ? error.message : String(error)}`)); } + if (config.get('autoStartMcpBridge', false)) { + startHttpBridgeProcess().catch(error => log(`Auto-start HTTP MCP bridge failed: ${error instanceof Error ? error.message : String(error)}`)); + } } async function runSequence(mode = 'auto') { const options = resolveOptions(); @@ -909,7 +938,7 @@ async function stopProcesses() { setStatusBarState('idle'); } } -function terminateProcess(proc, label) { +function terminateProcess(proc, label, afterStop) { if (!proc) { return Promise.resolve(); } @@ -917,14 +946,17 @@ function terminateProcess(proc, label) { let finished = false; let termTimer; let killTimer; - const cleanup = () => { + const clearTimers = () => { if (termTimer) clearTimeout(termTimer); if (killTimer) clearTimeout(killTimer); }; const finalize = (reason) => { if (finished) return; finished = true; - cleanup(); + clearTimers(); + if (typeof afterStop === 'function') { + afterStop(); + } if (proc === forceProcess) { forceProcess = undefined; } @@ -1100,6 +1132,17 @@ async function writeMcpConfig() { const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; const serverModeRaw = (settings.get('mcpServerMode') || 'bridge'); const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + const effectiveMode = + serverMode === 'bridge' + ? (transportMode === 'http' ? 'bridge-http' : 'bridge-stdio') + : (transportMode === 'http' ? 'direct-http' : 'direct-sse'); + log(`Context Engine Uploader: MCP wiring mode=${effectiveMode} (serverMode=${serverMode}, transportMode=${transportMode}).`); + if (effectiveMode === 'bridge-http') { + const bridgeUrl = resolveBridgeHttpUrl(); + if (bridgeUrl) { + log(`Context Engine Uploader: bridge HTTP endpoint ${bridgeUrl}`); + } + } let indexerUrl = (settings.get('mcpIndexerUrl') || 'http://localhost:8001/sse').trim(); let memoryUrl = (settings.get('mcpMemoryUrl') || 'http://localhost:8000/sse').trim(); @@ -1536,7 +1579,7 @@ async function scaffoldCtxConfigFiles(workspaceDir, collectionName) { } function deactivate() { disposeIndexedWatcher(); - return stopProcesses(); + return Promise.all([stopProcesses(), stopHttpBridgeProcess()]); } module.exports = { activate, @@ -1569,6 +1612,187 @@ function getDefaultWindsurfMcpPath() { return path.join(os.homedir(), '.codeium', 'windsurf', 'mcp_config.json'); } +function resolveHttpBridgeOptions() { + try { + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + const serverModeRaw = settings.get('mcpServerMode') || 'bridge'; + const transportModeRaw = settings.get('mcpTransportMode') || 'sse-remote'; + const serverMode = typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge'; + const transportMode = typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote'; + if (serverMode !== 'bridge') { + vscode.window.showWarningMessage('Context Engine Uploader: MCP server mode is not "bridge"; HTTP bridge will connect to raw endpoints.'); + } + if (transportMode !== 'http') { + log('Context Engine Uploader: MCP transport mode is not "http"; HTTP bridge will still start but downstream configs may expect SSE.'); + } + const workspacePath = getWorkspaceFolderPath() || (settings.get('targetPath') || '').trim(); + if (!workspacePath) { + vscode.window.showErrorMessage('Context Engine Uploader: open a workspace or set contextEngineUploader.targetPath before starting HTTP MCP bridge.'); + return undefined; + } + let indexerUrl = (settings.get('mcpIndexerUrl') || 'http://localhost:8001/sse').trim(); + let memoryUrl = (settings.get('mcpMemoryUrl') || '').trim(); + indexerUrl = normalizeBridgeUrl(indexerUrl); + memoryUrl = normalizeBridgeUrl(memoryUrl); + let port = Number(settings.get('mcpBridgePort') || 30810); + if (!Number.isFinite(port) || port <= 0) { + port = 30810; + } + return { + workspacePath, + indexerUrl, + memoryUrl, + port + }; + } catch (error) { + log(`Failed to resolve HTTP bridge options: ${error instanceof Error ? error.message : String(error)}`); + return undefined; + } +} + +function resolveBridgeHttpUrl() { + try { + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + let port = Number(settings.get('mcpBridgePort') || 30810); + if (!Number.isFinite(port) || port <= 0) { + port = 30810; + } + const hostname = '127.0.0.1'; + return `http://${hostname}:${port}/mcp`; + } catch (error) { + log(`Failed to resolve bridge HTTP URL: ${error instanceof Error ? error.message : String(error)}`); + return undefined; + } +} + +async function startHttpBridgeProcess() { + if (httpBridgeProcess) { + vscode.window.showInformationMessage(`Context Engine HTTP MCP bridge already running on port ${httpBridgePort || 'unknown'}.`); + return httpBridgePort; + } + const options = resolveHttpBridgeOptions(); + if (!options) { + return undefined; + } + const invocation = resolveBridgeCliInvocation(); + if (!invocation) { + vscode.window.showErrorMessage('Context Engine Uploader: unable to locate ctxce CLI for HTTP bridge.'); + return undefined; + } + const cliArgs = ['mcp-http-serve']; + if (options.workspacePath) { + cliArgs.push('--workspace', normalizeWorkspaceForBridge(options.workspacePath)); + } + if (options.indexerUrl) { + cliArgs.push('--indexer-url', options.indexerUrl); + } + if (options.memoryUrl) { + cliArgs.push('--memory-url', options.memoryUrl); + } + if (options.port) { + cliArgs.push('--port', String(options.port)); + } + const finalArgs = [...invocation.args, ...cliArgs]; + log(`Starting HTTP MCP bridge via ${invocation.command} ${finalArgs.join(' ')}`); + const child = spawn(invocation.command, finalArgs, { + cwd: options.workspacePath, + env: process.env + }); + httpBridgeProcess = child; + httpBridgePort = options.port; + httpBridgeWorkspace = options.workspacePath; + attachOutput(child, 'mcp-http'); + child.on('exit', (code, signal) => { + log(`HTTP MCP bridge exited with code ${code} signal ${signal || ''}`.trim()); + if (httpBridgeProcess === child) { + httpBridgeProcess = undefined; + httpBridgePort = undefined; + httpBridgeWorkspace = undefined; + } + }); + child.on('error', error => { + log(`HTTP MCP bridge process error: ${error instanceof Error ? error.message : String(error)}`); + if (httpBridgeProcess === child) { + httpBridgeProcess = undefined; + httpBridgePort = undefined; + httpBridgeWorkspace = undefined; + } + }); + vscode.window.showInformationMessage(`Context Engine HTTP MCP bridge listening on http://127.0.0.1:${options.port}/mcp`); + return options.port; +} + +function stopHttpBridgeProcess() { + if (!httpBridgeProcess) { + return Promise.resolve(); + } + return terminateProcess( + httpBridgeProcess, + 'mcp-http', + () => { + httpBridgeProcess = undefined; + httpBridgePort = undefined; + httpBridgeWorkspace = undefined; + } + ); +} + +async function handleHttpBridgeSettingsChanged() { + const config = vscode.workspace.getConfiguration('contextEngineUploader'); + const shouldRun = !!config.get('autoStartMcpBridge', false); + const wasRunning = !!httpBridgeProcess; + if (httpBridgeProcess) { + await stopHttpBridgeProcess(); + } + if (shouldRun || wasRunning) { + await startHttpBridgeProcess(); + } +} + +function resolveBridgeCliInvocation() { + const binPath = findLocalBridgeBin(); + if (binPath) { + const nodeExec = process.execPath || 'node'; + return { + command: nodeExec, + args: [binPath], + kind: 'local' + }; + } + const isWindows = process.platform === 'win32'; + if (isWindows) { + return { + command: 'cmd', + args: ['/c', 'npx', '@context-engine-bridge/context-engine-mcp-bridge'], + kind: 'npx' + }; + } + return { + command: 'npx', + args: ['@context-engine-bridge/context-engine-mcp-bridge'], + kind: 'npx' + }; +} + +function findLocalBridgeBin() { + try { + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + const configured = (settings.get('mcpBridgeBinPath') || '').trim(); + if (configured && fs.existsSync(configured)) { + return path.resolve(configured); + } + } catch (_) { + // ignore config lookup failures + } + + const envOverride = (process.env.CTXCE_BRIDGE_BIN || '').trim(); + if (envOverride && fs.existsSync(envOverride)) { + return path.resolve(envOverride); + } + + return undefined; +} + async function writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode, serverMode = 'bridge') { const configPath = path.join(root, '.mcp.json'); let config = { mcpServers: {} }; @@ -1593,8 +1817,21 @@ async function writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode, const mode = (typeof transportMode === 'string' ? transportMode.trim() : 'sse-remote') || 'sse-remote'; if (serverMode === 'bridge') { - const bridgeServer = buildBridgeServerConfig(root, indexerUrl, memoryUrl); - servers['context-engine'] = bridgeServer; + if (mode === 'http') { + const bridgeUrl = resolveBridgeHttpUrl(); + if (bridgeUrl) { + servers['context-engine'] = { + type: 'http', + url: bridgeUrl + }; + } else { + const bridgeServer = buildBridgeServerConfig(root, indexerUrl, memoryUrl); + servers['context-engine'] = bridgeServer; + } + } else { + const bridgeServer = buildBridgeServerConfig(root, indexerUrl, memoryUrl); + servers['context-engine'] = bridgeServer; + } delete servers['qdrant-indexer']; delete servers.memory; } else if (mode === 'http') { @@ -1678,8 +1915,20 @@ async function writeWindsurfMcpServers(configPath, indexerUrl, memoryUrl, transp const mode = (typeof transportMode === 'string' ? transportMode.trim() : 'sse-remote') || 'sse-remote'; if (serverMode === 'bridge') { - const bridgeServer = buildBridgeServerConfig(workspaceHint || '', indexerUrl, memoryUrl); - servers['context-engine'] = bridgeServer; + if (mode === 'http') { + const bridgeUrl = resolveBridgeHttpUrl(); + if (bridgeUrl) { + servers['context-engine'] = { + serverUrl: bridgeUrl + }; + } else { + const bridgeServer = buildBridgeServerConfig(workspaceHint || '', indexerUrl, memoryUrl); + servers['context-engine'] = bridgeServer; + } + } else { + const bridgeServer = buildBridgeServerConfig(workspaceHint || '', indexerUrl, memoryUrl); + servers['context-engine'] = bridgeServer; + } delete servers['qdrant-indexer']; delete servers.memory; } else if (mode === 'http') { diff --git a/vscode-extension/context-engine-uploader/package.json b/vscode-extension/context-engine-uploader/package.json index b1269b94..b315edde 100644 --- a/vscode-extension/context-engine-uploader/package.json +++ b/vscode-extension/context-engine-uploader/package.json @@ -164,13 +164,28 @@ "type": "string", "enum": ["sse-remote", "http"], "default": "http", - "description": "Transport mode for Claude/Windsurf MCP configs: SSE via mcp-remote (sse-remote) or direct HTTP /mcp endpoints (http)." + "description": "Transport mode for MCP wiring. When server mode is \"bridge\", sse-remote starts the stdio ctxce bridge while http points clients at the local HTTP bridge URL." + }, + "contextEngineUploader.autoStartMcpBridge": { + "type": "boolean", + "default": true, + "description": "When enabled, automatically start the local ctx-mcp-bridge HTTP server for the active workspace so IDE clients can connect over HTTP without manual commands." + }, + "contextEngineUploader.mcpBridgePort": { + "type": "number", + "default": 30810, + "description": "Port used for the local ctx-mcp-bridge HTTP server when auto-starting from the VS Code extension. Change if multiple IDE windows need parallel bridges." + }, + "contextEngineUploader.mcpBridgeLocalOnly": { + "type": "boolean", + "default": true, + "description": "Development toggle. When true (default) the extension only runs the local ctx-mcp-bridge/bin/ctxce.js. Set to false to fall back to published npm builds via npx." }, "contextEngineUploader.mcpServerMode": { "type": "string", "enum": ["bridge", "direct"], "default": "bridge", - "description": "Preferred MCP server wiring. \"bridge\" uses the @context-engine-bridge/context-engine-mcp-bridge CLI (recommended). \"direct\" writes separate Qdrant indexer and memory MCP entries like previous versions." + "description": "MCP wiring style. \"bridge\" uses the ctxce MCP bridge (stdio if transport=sse-remote, HTTP if transport=http). \"direct\" writes separate indexer/memory servers (stdio or HTTP depending on the transport setting)." }, "contextEngineUploader.mcpIndexerUrl": { "type": "string", From 94d74c79d9459668f6471b9de0f630d93bbb5182 Mon Sep 17 00:00:00 2001 From: Reese Date: Tue, 9 Dec 2025 20:22:54 +0000 Subject: [PATCH 12/55] vscode ext: Refactors MCP config and bridge management Improves MCP configuration and management by introducing server modes ("bridge" and "direct") and clarifying transport modes ("sse-remote" and "http"). This change provides more flexibility in how MCP servers are wired, enabling both bridged and direct connections with different transport options. It also automates the startup of the HTTP bridge when necessary and refreshes MCP configs after the bridge is ready. --- .../context-engine-uploader/README.md | 44 +++++- .../context-engine-uploader/extension.js | 137 +++++++++++++++--- .../context-engine-uploader/package.json | 14 +- 3 files changed, 170 insertions(+), 25 deletions(-) diff --git a/vscode-extension/context-engine-uploader/README.md b/vscode-extension/context-engine-uploader/README.md index 50df124a..89138292 100644 --- a/vscode-extension/context-engine-uploader/README.md +++ b/vscode-extension/context-engine-uploader/README.md @@ -21,9 +21,16 @@ Configuration - **Prompt+ decoder:** set `Context Engine Uploader: Decoder Url` (default `http://localhost:8081`, auto-appends `/completion`) to point at your local llama.cpp decoder. For Ollama, set it to `http://localhost:11434/api/chat`. Turn on `Use Gpu Decoder` to set `USE_GPU_DECODER=1` so ctx.py prefers the GPU llama.cpp sidecar. Prompt+ automatically runs the bundled `scripts/ctx.py` when an embedded copy is available, falling back to the workspace version if not. - **Claude/Windsurf MCP config:** - `MCP Indexer Url` and `MCP Memory Url` control the URLs written into the project-local `.mcp.json` (Claude) and Windsurf `mcp_config.json` when you run the `Write MCP Config` command. These URLs are used **literally** (e.g. `http://localhost:8001/sse` or `http://localhost:8003/mcp`). - - `MCP Transport Mode` (`contextEngineUploader.mcpTransportMode`) chooses how those URLs are wrapped: - - `sse-remote` (default): emit stdio configs that call `npx mcp-remote --transport sse-only`. - - `http`: emit direct HTTP MCP entries of the form `{ "type": "http", "url": "" }` for Claude/Windsurf. Use this when pointing at HTTP `/mcp` endpoints exposed by the Context-Engine MCP services. + - `MCP Server Mode` (`contextEngineUploader.mcpServerMode`) controls *what* servers are written: + - `bridge`: write a single `context-engine` server that talks to the `ctxce` MCP bridge. + - `direct`: write two servers, `qdrant-indexer` and `memory`, that talk directly to the configured URLs. + - `MCP Transport Mode` (`contextEngineUploader.mcpTransportMode`) controls *how* those servers talk: + - `sse-remote` (default): use stdio MCP processes behind an SSE tunnel (`bridge-stdio` / `direct-sse`). + - `http`: use HTTP MCP endpoints directly (`bridge-http` / `direct-http`). + - Common combinations: + - `bridge` + `sse-remote` → **bridge-stdio**: write a single `context-engine` stdio server that runs `ctxce mcp-serve` behind an SSE tunnel. + - `bridge` + `http` → **bridge-http**: write a single `context-engine` HTTP server that points at the local `ctxce mcp-http-serve` URL (e.g. `http://127.0.0.1:30810/mcp`). + - `direct` + `http` → **direct-http**: write separate HTTP servers for the indexer and memory MCP backends. - **MCP config on startup:** - `contextEngineUploader.autoWriteMcpConfigOnStartup` (default `false`) controls whether the extension automatically runs the same logic as `Write MCP Config` on activation. When enabled, it refreshes `.mcp.json`, Windsurf `mcp_config.json`, and the Claude hook (`.claude/settings.local.json`) to match your current settings and the installed extension version. If `scaffoldCtxConfig` is also `true`, this startup path will additionally scaffold/update `ctx_config.json` and `.env` as described below. - **CTX + GLM settings:** @@ -37,6 +44,33 @@ Configuration - The scaffolder also enforces CTX defaults (e.g., `MULTI_REPO_MODE=1`, `REFRAG_RUNTIME=glm`, `REFRAG_DECODER=1`) so the embedded `ctx.py` is ready for remote uploads, regardless of the “Use GLM Decoder” toggle. - `contextEngineUploader.surfaceQdrantCollectionHint` gates whether the Claude hook adds a hint line with the Qdrant collection ID when ctx is enhancing prompts. This setting is also respected when the extension writes `.claude/settings.local.json`. +MCP bridge (ctx-mcp-bridge) & MCP config lifecycle +--------------------------------------------------- +- The MCP bridge (`@context-engine-bridge/context-engine-mcp-bridge`, CLI `ctxce`) is a small local MCP server that fans out to two upstream MCP services: the Qdrant indexer and the memory/search backend. The VS Code extension can drive it in two ways: + - **Bridge stdio (`bridge-stdio`)** – a stdio MCP server (`ctxce mcp-serve`) wrapped behind an SSE tunnel. + - **Bridge HTTP (`bridge-http`)** – an HTTP MCP server (`ctxce mcp-http-serve`) listening on `http://127.0.0.1:/mcp`. +- Why use the bridge instead of two direct MCP entries? + - **Single server entry:** IDEs only need to register one MCP server (`context-engine`) instead of juggling separate `qdrant-indexer` and `memory` entries, avoiding coordination mistakes. + - **Shared session defaults:** the bridge loads `ctx_config.json` and injects collection name, repo metadata, and any other ctx defaults so every IDE window talks to the right collection without hand-editing `.mcp.json`. + - **Per-user credential isolation:** each IDE maintains its own MCP session while the bridge multiplexes upstream calls, so user preferences (future auth) remain per client even though the backend pair is shared. + - **Flexible transport:** stdio mode works everywhere (even when HTTP ports aren’t reachable), while HTTP mode keeps Claude/Windsurf happy when they want direct URLs; the extension automatically writes the right flavor. + - **Centralized logging & health:** when the bridge process runs once per workspace you get a single stream of logs (`Context Engine Upload` output) and a single port to probe for health checks instead of multiple MCP child processes per IDE. +- When you run **`Write MCP Config`**, the extension: + - Writes `.mcp.json` in the workspace for Claude Code. + - Optionally writes Windsurf’s `mcp_config.json` (when `mcpWindsurfEnabled=true`). + - Optionally scaffolds `ctx_config.json` + `.env` (when `scaffoldCtxConfig=true`). +- The effective wiring mode is determined by the two MCP settings: + - `mcpServerMode = bridge`, `mcpTransportMode = sse-remote` → **bridge-stdio**. + - `mcpServerMode = bridge`, `mcpTransportMode = http` → **bridge-http**. + - `mcpServerMode = direct`, `mcpTransportMode = sse-remote` → **direct-sse** (two stdio `mcp-remote` servers). + - `mcpServerMode = direct`, `mcpTransportMode = http` → **direct-http** (two HTTP servers, no bridge). +- In **bridge-stdio**, the configs run `ctxce mcp-serve` via `npx`, passing the workspace path (auto-detected from the uploader target path) plus `--indexer-url` and `--memory-url` derived from the MCP settings. +- In **bridge-http**, the extension can also **manage the bridge process**: + - `autoStartMcpBridge=true` and `mcpServerMode='bridge'` with `mcpTransportMode='http'` → the extension starts `ctxce mcp-http-serve` in the background for the active workspace using `mcpBridgePort`. + - The resulting HTTP URL (`http://127.0.0.1:/mcp`) is written into `.mcp.json` and Windsurf’s `mcp_config.json` as the `context-engine` server URL. + - In **stdio or direct modes**, the HTTP bridge is **not** auto-started; only the explicit `Start MCP HTTP Bridge` command will launch it. +- Bridge settings are **workspace-scoped**, so different workspaces can choose different modes and ports (e.g., one workspace using stdio bridge, another using HTTP bridge on a different port). + Workspace-level ctx integration ------------------------------- - The VSIX bundles an `env.example` template plus the ctx hook/CLI so you can dogfood the workflow without copying files manually. @@ -51,9 +85,11 @@ Commands -------- - Command Palette → “Context Engine Uploader” exposes Start/Stop/Restart/Index Codebase and Prompt+ (unicorn) rewrite commands. - Status-bar button (`Index Codebase`) mirrors Start/Stop/Restart/Index status, while the `Prompt+` status button runs the ctx rewrite command on the current selection. -- `Context Engine Uploader: Write MCP Config (.mcp.json)` writes or updates a project-local `.mcp.json` with MCP server entries for the Qdrant indexer and memory/search endpoints, using the configured MCP URLs. +- `Context Engine Uploader: Write MCP Config (.mcp.json)` writes or updates a project-local `.mcp.json` (plus Windsurf `mcp_config.json` when enabled) using the currently selected bridge/direct + transport modes. If bridge-http is required and not yet running, the extension starts `ctxce mcp-http-serve` before writing configs. - `Context Engine Uploader: Write CTX Config (ctx_config.json/.env)` scaffolds the ctx config + env files as described above. This command runs automatically after `Write MCP Config` if scaffolding is enabled, but it is also exposed in the Command Palette for manual use. - `Context Engine Uploader: Upload Git History (force sync bundle)` triggers a one-off force sync using the configured git history settings, producing a bundle that includes a `metadata/git_history.json` manifest for remote lineage ingestion. +- `Context Engine Uploader: Start MCP HTTP Bridge` launches `ctxce mcp-http-serve` using the workspace’s resolved target path, MCP URLs, and configured `mcpBridgePort`. Use this when you want to run the HTTP bridge manually (e.g., testing unpublished builds or sharing a port across IDEs). +- `Context Engine Uploader: Stop MCP HTTP Bridge` gracefully terminates a running HTTP bridge process. Logs ---- diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index 6615a98a..5930e64b 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -20,6 +20,7 @@ let pythonOverridePath; let httpBridgeProcess; let httpBridgePort; let httpBridgeWorkspace; +let pendingBridgeConfigTimer; const REQUIRED_PYTHON_MODULES = ['requests', 'urllib3', 'charset_normalizer']; const DEFAULT_CONTAINER_ROOT = '/work'; // const CLAUDE_HOOK_COMMAND = '/home/coder/project/Context-Engine/ctx-hook-simple.sh'; @@ -173,7 +174,15 @@ function activate(context) { writeCtxConfig().catch(error => log(`CTX config auto-scaffold on activation failed: ${error instanceof Error ? error.message : String(error)}`)); } if (config.get('autoStartMcpBridge', false)) { - startHttpBridgeProcess().catch(error => log(`Auto-start HTTP MCP bridge failed: ${error instanceof Error ? error.message : String(error)}`)); + const transportModeRaw = config.get('mcpTransportMode') || 'sse-remote'; + const serverModeRaw = config.get('mcpServerMode') || 'bridge'; + const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; + const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + if (requiresHttpBridge(serverMode, transportMode)) { + startHttpBridgeProcess().catch(error => log(`Auto-start HTTP MCP bridge failed: ${error instanceof Error ? error.message : String(error)}`)); + } else { + log('Context Engine Uploader: autoStartMcpBridge is enabled, but current MCP wiring does not use the HTTP bridge; skipping auto-start.'); + } } } async function runSequence(mode = 'auto') { @@ -284,7 +293,7 @@ function resolveOptions() { startWatchAfterForce }; } -function getTargetPath(config) { +function resolveTargetPathFromConfig(config) { let inspected; try { if (typeof config.inspect === 'function') { @@ -294,6 +303,22 @@ function getTargetPath(config) { inspected = undefined; } let targetPath = (config.get('targetPath') || '').trim(); + const metadata = inspected || {}; + if (targetPath) { + return { path: targetPath, inspected: metadata }; + } + const folderPath = getWorkspaceFolderPath(); + if (!folderPath) { + return { path: undefined, inspected: metadata }; + } + const autoTarget = detectDefaultTargetPath(folderPath); + return { path: autoTarget, inspected: metadata, inferred: true }; +} + +function getTargetPath(config) { + const result = resolveTargetPathFromConfig(config); + const targetPath = result.path; + const inspected = result.inspected; if (inspected && targetPath) { let sourceLabel = 'default'; if (inspected.workspaceFolderValue !== undefined) { @@ -312,15 +337,9 @@ function getTargetPath(config) { updateStatusBarTooltip(targetPath); return targetPath; } - const folderPath = getWorkspaceFolderPath(); - if (!folderPath) { - vscode.window.showErrorMessage('Context Engine Uploader: open a folder or set contextEngineUploader.targetPath.'); - updateStatusBarTooltip(); - return undefined; - } - const autoTarget = detectDefaultTargetPath(folderPath); - updateStatusBarTooltip(autoTarget); - return autoTarget; + vscode.window.showErrorMessage('Context Engine Uploader: open a folder or set contextEngineUploader.targetPath.'); + updateStatusBarTooltip(); + return undefined; } function saveTargetPath(config, targetPath) { const hasWorkspace = vscode.workspace.workspaceFolders && vscode.workspace.workspaceFolders.length; @@ -398,6 +417,47 @@ function detectDefaultTargetPath(workspaceFolderPath) { return workspaceFolderPath; } } + +function resolveBridgeWorkspacePath() { + try { + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + const target = getTargetPath(settings); + if (target) { + return path.resolve(target); + } + } catch (error) { + log(`Context Engine Uploader: failed to resolve bridge workspace path via getTargetPath: ${error instanceof Error ? error.message : String(error)}`); + } + const fallbackFolder = getWorkspaceFolderPath(); + if (!fallbackFolder) { + return undefined; + } + try { + const autoTarget = detectDefaultTargetPath(fallbackFolder); + return autoTarget ? path.resolve(autoTarget) : path.resolve(fallbackFolder); + } catch (error) { + log(`Context Engine Uploader: failed fallback bridge workspace path detection: ${error instanceof Error ? error.message : String(error)}`); + return undefined; + } +} + +function scheduleMcpConfigRefreshAfterBridge(delayMs = 1500) { + try { + if (pendingBridgeConfigTimer) { + clearTimeout(pendingBridgeConfigTimer); + pendingBridgeConfigTimer = undefined; + } + pendingBridgeConfigTimer = setTimeout(() => { + pendingBridgeConfigTimer = undefined; + log('Context Engine Uploader: HTTP bridge ready; refreshing MCP configs.'); + writeMcpConfig().catch(error => { + log(`Context Engine Uploader: MCP config refresh after bridge start failed: ${error instanceof Error ? error.message : String(error)}`); + }); + }, delayMs); + } catch (error) { + log(`Context Engine Uploader: failed to schedule MCP config refresh: ${error instanceof Error ? error.message : String(error)}`); + } +} function ensureTargetPathConfigured() { const config = vscode.workspace.getConfiguration('contextEngineUploader'); const current = (config.get('targetPath') || '').trim(); @@ -605,6 +665,23 @@ async function detectSystemPython() { } return undefined; } + +function requiresHttpBridge(serverMode, transportMode) { + return serverMode === 'bridge' && transportMode === 'http'; +} + +async function ensureHttpBridgeReadyForConfigs() { + try { + if (httpBridgeProcess) { + return true; + } + await startHttpBridgeProcess(); + return !!httpBridgeProcess; + } catch (error) { + log(`Failed to ensure HTTP bridge is ready: ${error instanceof Error ? error.message : String(error)}`); + return false; + } +} function setStatusBarState(mode) { if (!statusBarItem) { return; @@ -1132,6 +1209,19 @@ async function writeMcpConfig() { const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; const serverModeRaw = (settings.get('mcpServerMode') || 'bridge'); const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + const needsHttpBridge = requiresHttpBridge(serverMode, transportMode); + const bridgeWasRunning = !!httpBridgeProcess; + if (needsHttpBridge) { + const ready = await ensureHttpBridgeReadyForConfigs(); + if (!ready) { + vscode.window.showErrorMessage('Context Engine Uploader: HTTP MCP bridge failed to start; MCP config not updated.'); + return; + } + if (!bridgeWasRunning && httpBridgeProcess) { + log('Context Engine Uploader: HTTP MCP bridge launching; delaying MCP config write until bridge signals ready.'); + return; + } + } const effectiveMode = serverMode === 'bridge' ? (transportMode === 'http' ? 'bridge-http' : 'bridge-stdio') @@ -1625,7 +1715,7 @@ function resolveHttpBridgeOptions() { if (transportMode !== 'http') { log('Context Engine Uploader: MCP transport mode is not "http"; HTTP bridge will still start but downstream configs may expect SSE.'); } - const workspacePath = getWorkspaceFolderPath() || (settings.get('targetPath') || '').trim(); + const workspacePath = resolveBridgeWorkspacePath(); if (!workspacePath) { vscode.window.showErrorMessage('Context Engine Uploader: open a workspace or set contextEngineUploader.targetPath before starting HTTP MCP bridge.'); return undefined; @@ -1719,6 +1809,7 @@ async function startHttpBridgeProcess() { } }); vscode.window.showInformationMessage(`Context Engine HTTP MCP bridge listening on http://127.0.0.1:${options.port}/mcp`); + scheduleMcpConfigRefreshAfterBridge(); return options.port; } @@ -1745,7 +1836,15 @@ async function handleHttpBridgeSettingsChanged() { await stopHttpBridgeProcess(); } if (shouldRun || wasRunning) { - await startHttpBridgeProcess(); + const transportModeRaw = config.get('mcpTransportMode') || 'sse-remote'; + const serverModeRaw = config.get('mcpServerMode') || 'bridge'; + const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; + const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + if (requiresHttpBridge(serverMode, transportMode)) { + await startHttpBridgeProcess(); + } else { + log('Context Engine Uploader: HTTP bridge settings changed, but current MCP wiring does not use the HTTP bridge; not restarting HTTP bridge.'); + } } } @@ -1794,7 +1893,8 @@ function findLocalBridgeBin() { } async function writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode, serverMode = 'bridge') { - const configPath = path.join(root, '.mcp.json'); + const bridgeWorkspace = resolveBridgeWorkspacePath(); + const configPath = path.join(bridgeWorkspace || root, '.mcp.json'); let config = { mcpServers: {} }; if (fs.existsSync(configPath)) { try { @@ -1825,11 +1925,11 @@ async function writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode, url: bridgeUrl }; } else { - const bridgeServer = buildBridgeServerConfig(root, indexerUrl, memoryUrl); + const bridgeServer = buildBridgeServerConfig(bridgeWorkspace || root, indexerUrl, memoryUrl); servers['context-engine'] = bridgeServer; } } else { - const bridgeServer = buildBridgeServerConfig(root, indexerUrl, memoryUrl); + const bridgeServer = buildBridgeServerConfig(bridgeWorkspace || root, indexerUrl, memoryUrl); servers['context-engine'] = bridgeServer; } delete servers['qdrant-indexer']; @@ -1915,6 +2015,7 @@ async function writeWindsurfMcpServers(configPath, indexerUrl, memoryUrl, transp const mode = (typeof transportMode === 'string' ? transportMode.trim() : 'sse-remote') || 'sse-remote'; if (serverMode === 'bridge') { + const bridgeWorkspace = resolveBridgeWorkspacePath() || workspaceHint || ''; if (mode === 'http') { const bridgeUrl = resolveBridgeHttpUrl(); if (bridgeUrl) { @@ -1922,11 +2023,11 @@ async function writeWindsurfMcpServers(configPath, indexerUrl, memoryUrl, transp serverUrl: bridgeUrl }; } else { - const bridgeServer = buildBridgeServerConfig(workspaceHint || '', indexerUrl, memoryUrl); + const bridgeServer = buildBridgeServerConfig(bridgeWorkspace, indexerUrl, memoryUrl); servers['context-engine'] = bridgeServer; } } else { - const bridgeServer = buildBridgeServerConfig(workspaceHint || '', indexerUrl, memoryUrl); + const bridgeServer = buildBridgeServerConfig(bridgeWorkspace, indexerUrl, memoryUrl); servers['context-engine'] = bridgeServer; } delete servers['qdrant-indexer']; diff --git a/vscode-extension/context-engine-uploader/package.json b/vscode-extension/context-engine-uploader/package.json index b315edde..c0d93f17 100644 --- a/vscode-extension/context-engine-uploader/package.json +++ b/vscode-extension/context-engine-uploader/package.json @@ -164,12 +164,16 @@ "type": "string", "enum": ["sse-remote", "http"], "default": "http", - "description": "Transport mode for MCP wiring. When server mode is \"bridge\", sse-remote starts the stdio ctxce bridge while http points clients at the local HTTP bridge URL." + "description": "Transport layer for MCP servers. 'sse-remote' runs stdio MCP processes behind an SSE tunnel; 'http' uses direct HTTP MCP endpoints. Combined with mcpServerMode this yields four modes: bridge-stdio, bridge-http, direct-sse, direct-http.", + "enumDescriptions": [ + "Use stdio MCP processes behind an SSE tunnel (bridge-stdio / direct-sse).", + "Use HTTP MCP endpoints directly (bridge-http / direct-http)." + ] }, "contextEngineUploader.autoStartMcpBridge": { "type": "boolean", "default": true, - "description": "When enabled, automatically start the local ctx-mcp-bridge HTTP server for the active workspace so IDE clients can connect over HTTP without manual commands." + "description": "When enabled and mcpServerMode='bridge' with mcpTransportMode='http', automatically start the local ctx-mcp-bridge HTTP server for the active workspace so IDE clients can connect over HTTP without manual commands. Has no effect in stdio/direct modes." }, "contextEngineUploader.mcpBridgePort": { "type": "number", @@ -185,7 +189,11 @@ "type": "string", "enum": ["bridge", "direct"], "default": "bridge", - "description": "MCP wiring style. \"bridge\" uses the ctxce MCP bridge (stdio if transport=sse-remote, HTTP if transport=http). \"direct\" writes separate indexer/memory servers (stdio or HTTP depending on the transport setting)." + "description": "MCP wiring style. 'bridge' writes a single 'context-engine' server that uses the ctxce MCP bridge (stdio when mcpTransportMode='sse-remote', HTTP when 'http'). 'direct' writes two servers ('qdrant-indexer' and 'memory'), using stdio when mcpTransportMode='sse-remote' and HTTP when 'http'.", + "enumDescriptions": [ + "Single 'context-engine' server powered by the ctxce bridge (stdio or HTTP depending on mcpTransportMode).", + "Two separate servers: 'qdrant-indexer' and 'memory' (stdio or HTTP depending on mcpTransportMode)." + ] }, "contextEngineUploader.mcpIndexerUrl": { "type": "string", From 3c1ce336a7628819d8c414b4dd98ed0ec2846de9 Mon Sep 17 00:00:00 2001 From: Reese Date: Tue, 9 Dec 2025 21:42:29 +0000 Subject: [PATCH 13/55] feat(ctx-mcp-bridge): Add CTXCE_TOOL_TIMEOUT_MSEC env var for tool timeouts --- ctx-mcp-bridge/src/mcpServer.js | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/ctx-mcp-bridge/src/mcpServer.js b/ctx-mcp-bridge/src/mcpServer.js index 5da6d5d7..bd152ed2 100644 --- a/ctx-mcp-bridge/src/mcpServer.js +++ b/ctx-mcp-bridge/src/mcpServer.js @@ -94,6 +94,22 @@ function withTimeout(promise, ms, label) { }); } +function getBridgeToolTimeoutMs() { + try { + const raw = process.env.CTXCE_TOOL_TIMEOUT_MSEC; + if (!raw) { + return 300000; + } + const parsed = Number.parseInt(String(raw), 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + return 300000; + } + return parsed; + } catch { + return 300000; + } +} + function selectClientForTool(name, indexerClient, memoryClient) { if (!name) { return indexerClient; @@ -300,10 +316,15 @@ async function createBridgeServer(options) { throw new Error(`Tool ${name} not available on any configured MCP server`); } - const result = await targetClient.callTool({ - name, - arguments: args, - }); + const timeoutMs = getBridgeToolTimeoutMs(); + const result = await targetClient.callTool( + { + name, + arguments: args, + }, + undefined, + { timeout: timeoutMs }, + ); return result; }); From 9b54f907f9263522f1f9831134d8b31602fcabe8 Mon Sep 17 00:00:00 2001 From: Reese Date: Tue, 9 Dec 2025 21:42:29 +0000 Subject: [PATCH 14/55] feat(ctx-mcp-bridge): Exit bridge process when stdin is closed --- ctx-mcp-bridge/src/mcpServer.js | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ctx-mcp-bridge/src/mcpServer.js b/ctx-mcp-bridge/src/mcpServer.js index bd152ed2..ea04713a 100644 --- a/ctx-mcp-bridge/src/mcpServer.js +++ b/ctx-mcp-bridge/src/mcpServer.js @@ -335,6 +335,27 @@ export async function runMcpServer(options) { const server = await createBridgeServer(options); const transport = new StdioServerTransport(); await server.connect(transport); + + const exitOnStdinClose = process.env.CTXCE_EXIT_ON_STDIN_CLOSE !== "0"; + if (exitOnStdinClose) { + const handleStdioClosed = () => { + try { + debugLog("[ctxce] Stdio transport closed; exiting MCP bridge process."); + } catch { + // ignore + } + // Allow any in-flight logs to flush, then exit. + setTimeout(() => { + process.exit(0); + }, 10).unref(); + }; + + if (process.stdin && typeof process.stdin.on === "function") { + process.stdin.on("end", handleStdioClosed); + process.stdin.on("close", handleStdioClosed); + process.stdin.on("error", handleStdioClosed); + } + } } export async function runHttpMcpServer(options) { From 64a90a187886534800c3b56b9d777f087c11a70f Mon Sep 17 00:00:00 2001 From: Reese Date: Tue, 9 Dec 2025 21:42:29 +0000 Subject: [PATCH 15/55] feat(vscode-extension): Allow configuring MCP bridge binary path and local-only execution --- .../context-engine-uploader/extension.js | 40 +++++++++---------- .../context-engine-uploader/package.json | 7 +++- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index 5930e64b..70f10544 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -127,6 +127,8 @@ function activate(context) { if ( event.affectsConfiguration('contextEngineUploader.autoStartMcpBridge') || event.affectsConfiguration('contextEngineUploader.mcpBridgePort') || + event.affectsConfiguration('contextEngineUploader.mcpBridgeBinPath') || + event.affectsConfiguration('contextEngineUploader.mcpBridgeLocalOnly') || event.affectsConfiguration('contextEngineUploader.mcpIndexerUrl') || event.affectsConfiguration('contextEngineUploader.mcpMemoryUrl') || event.affectsConfiguration('contextEngineUploader.mcpServerMode') || @@ -1167,11 +1169,8 @@ function normalizeWorkspaceForBridge(workspacePath) { } function buildBridgeServerConfig(workspacePath, indexerUrl, memoryUrl) { - const isWindows = process.platform === 'win32'; - const args = [ - '@context-engine-bridge/context-engine-mcp-bridge', - 'mcp-serve' - ]; + const invocation = resolveBridgeCliInvocation(); + const args = [...invocation.args, 'mcp-serve']; if (workspacePath) { args.push('--workspace', normalizeWorkspaceForBridge(workspacePath)); } @@ -1181,15 +1180,8 @@ function buildBridgeServerConfig(workspacePath, indexerUrl, memoryUrl) { if (memoryUrl) { args.push('--memory-url', memoryUrl); } - if (isWindows) { - return { - command: 'cmd', - args: ['/c', 'npx', ...args], - env: {} - }; - } return { - command: 'npx', + command: invocation.command, args, env: {} }; @@ -1851,9 +1843,8 @@ async function handleHttpBridgeSettingsChanged() { function resolveBridgeCliInvocation() { const binPath = findLocalBridgeBin(); if (binPath) { - const nodeExec = process.execPath || 'node'; return { - command: nodeExec, + command: 'node', args: [binPath], kind: 'local' }; @@ -1874,14 +1865,23 @@ function resolveBridgeCliInvocation() { } function findLocalBridgeBin() { + let localOnly = true; + let configured = ''; try { const settings = vscode.workspace.getConfiguration('contextEngineUploader'); - const configured = (settings.get('mcpBridgeBinPath') || '').trim(); - if (configured && fs.existsSync(configured)) { - return path.resolve(configured); - } + localOnly = settings.get('mcpBridgeLocalOnly', true); + configured = (settings.get('mcpBridgeBinPath') || '').trim(); } catch (_) { - // ignore config lookup failures + // ignore config lookup failures and fall back to env/npx behavior + } + + // When local-only is disabled, skip local resolution and always fall back to npx + if (localOnly === false) { + return undefined; + } + + if (configured && fs.existsSync(configured)) { + return path.resolve(configured); } const envOverride = (process.env.CTXCE_BRIDGE_BIN || '').trim(); diff --git a/vscode-extension/context-engine-uploader/package.json b/vscode-extension/context-engine-uploader/package.json index c0d93f17..a670e55f 100644 --- a/vscode-extension/context-engine-uploader/package.json +++ b/vscode-extension/context-engine-uploader/package.json @@ -180,10 +180,15 @@ "default": 30810, "description": "Port used for the local ctx-mcp-bridge HTTP server when auto-starting from the VS Code extension. Change if multiple IDE windows need parallel bridges." }, + "contextEngineUploader.mcpBridgeBinPath": { + "type": "string", + "default": "", + "description": "Optional path to the ctxce CLI binary/script to use for the MCP bridge in local development. When set and the file exists, the extension runs this path via the current Node executable instead of using CTXCE_BRIDGE_BIN or 'npx @context-engine-bridge/context-engine-mcp-bridge'." + }, "contextEngineUploader.mcpBridgeLocalOnly": { "type": "boolean", "default": true, - "description": "Development toggle. When true (default) the extension only runs the local ctx-mcp-bridge/bin/ctxce.js. Set to false to fall back to published npm builds via npx." + "description": "Development toggle. When true (default) the extension prefers local bridge binaries resolved from mcpBridgeBinPath or CTXCE_BRIDGE_BIN before falling back to the published npm build via npx." }, "contextEngineUploader.mcpServerMode": { "type": "string", From 17bbe1ba2e1766286a694ac7f4b2ed4d10213e7c Mon Sep 17 00:00:00 2001 From: Reese Date: Tue, 9 Dec 2025 21:43:37 +0000 Subject: [PATCH 16/55] MCP Bridge: version bump --- ctx-mcp-bridge/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctx-mcp-bridge/package.json b/ctx-mcp-bridge/package.json index cddce66f..cf16bc82 100644 --- a/ctx-mcp-bridge/package.json +++ b/ctx-mcp-bridge/package.json @@ -1,6 +1,6 @@ { "name": "@context-engine-bridge/context-engine-mcp-bridge", - "version": "0.0.3", + "version": "0.0.4", "description": "Context Engine MCP bridge (http/stdio proxy combining indexer + memory servers)", "bin": { "ctxce": "bin/ctxce.js", From 74d75aa83f5e3ab9a2feb78ab5f13ff5fd9431cf Mon Sep 17 00:00:00 2001 From: Reese Date: Wed, 10 Dec 2025 17:59:27 +0000 Subject: [PATCH 17/55] Bridge: Handles MCP session errors with client reinitialization Addresses potential issues where the MCP bridge encounters session errors, such as an expired or invalid session, by implementing a reinitialization mechanism. This ensures that the bridge attempts to re-establish connections to the remote MCP clients upon detecting a session-related error. This prevents the bridge from becoming unusable in cases where the underlying sessions expire or become invalidated. --- ctx-mcp-bridge/src/mcpServer.js | 191 ++++++++++++++++++++++---------- 1 file changed, 132 insertions(+), 59 deletions(-) diff --git a/ctx-mcp-bridge/src/mcpServer.js b/ctx-mcp-bridge/src/mcpServer.js index ea04713a..b9aac0dc 100644 --- a/ctx-mcp-bridge/src/mcpServer.js +++ b/ctx-mcp-bridge/src/mcpServer.js @@ -120,6 +120,25 @@ function selectClientForTool(name, indexerClient, memoryClient) { } return indexerClient; } + +function isSessionError(error) { + try { + const msg = + (error && typeof error.message === "string" && error.message) || + (typeof error === "string" ? error : String(error || "")); + if (!msg) { + return false; + } + return ( + msg.includes("No valid session ID") || + msg.includes("Mcp-Session-Id header is required") || + msg.includes("Server not initialized") || + msg.includes("Session not found") + ); + } catch { + return false; + } +} // MCP stdio server implemented using the official MCP TypeScript SDK. // Acts as a low-level proxy for tools, forwarding tools/list and tools/call // to the remote qdrant-indexer MCP server while adding a local `ping` tool. @@ -162,52 +181,8 @@ async function createBridgeServer(options) { ); } - // High-level MCP client for the remote HTTP /mcp indexer - const indexerTransport = new StreamableHTTPClientTransport(indexerUrl); - const indexerClient = new Client( - { - name: "ctx-context-engine-bridge-http-client", - version: "0.0.1", - }, - { - capabilities: { - tools: {}, - resources: {}, - prompts: {}, - }, - }, - ); - - try { - await indexerClient.connect(indexerTransport); - } catch (err) { - debugLog("[ctxce] Failed to connect MCP HTTP client to indexer: " + String(err)); - } - + let indexerClient = null; let memoryClient = null; - if (memoryUrl) { - try { - const memoryTransport = new StreamableHTTPClientTransport(memoryUrl); - memoryClient = new Client( - { - name: "ctx-context-engine-bridge-memory-client", - version: "0.0.1", - }, - { - capabilities: { - tools: {}, - resources: {}, - prompts: {}, - }, - }, - ); - await memoryClient.connect(memoryTransport); - debugLog(`[ctxce] Connected memory MCP client: ${memoryUrl}`); - } catch (err) { - debugLog("[ctxce] Failed to connect memory MCP client: " + String(err)); - memoryClient = null; - } - } // Derive a simple session identifier for this bridge process. In the // future this can be made user-aware (e.g. from auth), but for now we @@ -229,13 +204,81 @@ async function createBridgeServer(options) { defaultsPayload.under = defaultUnder; } - if (Object.keys(defaultsPayload).length > 1) { - await sendSessionDefaults(indexerClient, defaultsPayload, "indexer"); - if (memoryClient) { - await sendSessionDefaults(memoryClient, defaultsPayload, "memory"); + async function initializeRemoteClients(forceRecreate = false) { + if (!forceRecreate && indexerClient) { + return; + } + + if (forceRecreate) { + try { + debugLog("[ctxce] Reinitializing remote MCP clients after session error."); + } catch { + // ignore logging failures + } + } + + let nextIndexerClient = null; + try { + const indexerTransport = new StreamableHTTPClientTransport(indexerUrl); + const client = new Client( + { + name: "ctx-context-engine-bridge-http-client", + version: "0.0.1", + }, + { + capabilities: { + tools: {}, + resources: {}, + prompts: {}, + }, + }, + ); + await client.connect(indexerTransport); + nextIndexerClient = client; + } catch (err) { + debugLog("[ctxce] Failed to connect MCP HTTP client to indexer: " + String(err)); + nextIndexerClient = null; + } + + let nextMemoryClient = null; + if (memoryUrl) { + try { + const memoryTransport = new StreamableHTTPClientTransport(memoryUrl); + const client = new Client( + { + name: "ctx-context-engine-bridge-memory-client", + version: "0.0.1", + }, + { + capabilities: { + tools: {}, + resources: {}, + prompts: {}, + }, + }, + ); + await client.connect(memoryTransport); + debugLog(`[ctxce] Connected memory MCP client: ${memoryUrl}`); + nextMemoryClient = client; + } catch (err) { + debugLog("[ctxce] Failed to connect memory MCP client: " + String(err)); + nextMemoryClient = null; + } + } + + indexerClient = nextIndexerClient; + memoryClient = nextMemoryClient; + + if (Object.keys(defaultsPayload).length > 1 && indexerClient) { + await sendSessionDefaults(indexerClient, defaultsPayload, "indexer"); + if (memoryClient) { + await sendSessionDefaults(memoryClient, defaultsPayload, "memory"); + } } } + await initializeRemoteClients(false); + const server = new Server( // TODO: marked as depreciated { name: "ctx-context-engine-bridge", @@ -253,6 +296,10 @@ async function createBridgeServer(options) { let remote; try { debugLog("[ctxce] tools/list: fetching tools from indexer"); + await initializeRemoteClients(false); + if (!indexerClient) { + throw new Error("Indexer MCP client not initialized"); + } remote = await withTimeout( indexerClient.listTools(), 10000, @@ -311,21 +358,47 @@ async function createBridgeServer(options) { return indexerResult; } - const targetClient = selectClientForTool(name, indexerClient, memoryClient); + await initializeRemoteClients(false); + + let targetClient = selectClientForTool(name, indexerClient, memoryClient); if (!targetClient) { throw new Error(`Tool ${name} not available on any configured MCP server`); } const timeoutMs = getBridgeToolTimeoutMs(); - const result = await targetClient.callTool( - { - name, - arguments: args, - }, - undefined, - { timeout: timeoutMs }, - ); - return result; + try { + const result = await targetClient.callTool( + { + name, + arguments: args, + }, + undefined, + { timeout: timeoutMs }, + ); + return result; + } catch (err) { + if (isSessionError(err)) { + debugLog( + "[ctxce] tools/call: detected remote MCP session error; reinitializing clients and retrying once: " + + String(err), + ); + await initializeRemoteClients(true); + targetClient = selectClientForTool(name, indexerClient, memoryClient); + if (!targetClient) { + throw err; + } + const retryResult = await targetClient.callTool( + { + name, + arguments: args, + }, + undefined, + { timeout: timeoutMs }, + ); + return retryResult; + } + throw err; + } }); return server; From 9338d4069f400194b2fbb0202abadc6e47b8b208 Mon Sep 17 00:00:00 2001 From: Reese Date: Wed, 10 Dec 2025 18:30:15 +0000 Subject: [PATCH 18/55] Bridge: Improves tool call reliability with retries Enhances the resilience of tool calls by adding retry logic for transient errors. Introduces configurable retry attempts and delay via environment variables. Also detects transient errors based on message content and error codes. This change ensures that temporary network issues or service unavailability do not lead to immediate tool call failures, improving the overall stability of the system. --- ctx-mcp-bridge/package.json | 2 +- ctx-mcp-bridge/src/mcpServer.js | 164 ++++++++++++++++++++++++++------ 2 files changed, 136 insertions(+), 30 deletions(-) diff --git a/ctx-mcp-bridge/package.json b/ctx-mcp-bridge/package.json index cf16bc82..3bcb4244 100644 --- a/ctx-mcp-bridge/package.json +++ b/ctx-mcp-bridge/package.json @@ -1,6 +1,6 @@ { "name": "@context-engine-bridge/context-engine-mcp-bridge", - "version": "0.0.4", + "version": "0.0.5", "description": "Context Engine MCP bridge (http/stdio proxy combining indexer + memory servers)", "bin": { "ctxce": "bin/ctxce.js", diff --git a/ctx-mcp-bridge/src/mcpServer.js b/ctx-mcp-bridge/src/mcpServer.js index b9aac0dc..f967f83a 100644 --- a/ctx-mcp-bridge/src/mcpServer.js +++ b/ctx-mcp-bridge/src/mcpServer.js @@ -139,6 +139,99 @@ function isSessionError(error) { return false; } } + +function getBridgeRetryAttempts() { + try { + const raw = process.env.CTXCE_TOOL_RETRY_ATTEMPTS; + if (!raw) { + return 2; + } + const parsed = Number.parseInt(String(raw), 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + return 1; + } + return parsed; + } catch { + return 2; + } +} + +function getBridgeRetryDelayMs() { + try { + const raw = process.env.CTXCE_TOOL_RETRY_DELAY_MSEC; + if (!raw) { + return 200; + } + const parsed = Number.parseInt(String(raw), 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return 0; + } + return parsed; + } catch { + return 200; + } +} + +function isTransientToolError(error) { + try { + const msg = + (error && typeof error.message === "string" && error.message) || + (typeof error === "string" ? error : String(error || "")); + if (!msg) { + return false; + } + const lower = msg.toLowerCase(); + + if ( + lower.includes("timed out") || + lower.includes("timeout") || + lower.includes("time-out") + ) { + return true; + } + + if ( + lower.includes("econnreset") || + lower.includes("econnrefused") || + lower.includes("etimedout") || + lower.includes("enotfound") || + lower.includes("ehostunreach") || + lower.includes("enetunreach") + ) { + return true; + } + + if ( + lower.includes("bad gateway") || + lower.includes("gateway timeout") || + lower.includes("service unavailable") || + lower.includes(" 502 ") || + lower.includes(" 503 ") || + lower.includes(" 504 ") + ) { + return true; + } + + if (lower.includes("network error")) { + return true; + } + + if (typeof error.code === "number" && error.code === -32001 && !isSessionError(error)) { + return true; + } + if ( + typeof error.code === "string" && + error.code.toLowerCase && + error.code.toLowerCase().includes("timeout") + ) { + return true; + } + + return false; + } catch { + return false; + } +} // MCP stdio server implemented using the official MCP TypeScript SDK. // Acts as a low-level proxy for tools, forwarding tools/list and tools/call // to the remote qdrant-indexer MCP server while adding a local `ping` tool. @@ -360,34 +453,24 @@ async function createBridgeServer(options) { await initializeRemoteClients(false); - let targetClient = selectClientForTool(name, indexerClient, memoryClient); - if (!targetClient) { - throw new Error(`Tool ${name} not available on any configured MCP server`); - } - const timeoutMs = getBridgeToolTimeoutMs(); - try { - const result = await targetClient.callTool( - { - name, - arguments: args, - }, - undefined, - { timeout: timeoutMs }, - ); - return result; - } catch (err) { - if (isSessionError(err)) { - debugLog( - "[ctxce] tools/call: detected remote MCP session error; reinitializing clients and retrying once: " + - String(err), - ); - await initializeRemoteClients(true); - targetClient = selectClientForTool(name, indexerClient, memoryClient); - if (!targetClient) { - throw err; - } - const retryResult = await targetClient.callTool( + const maxAttempts = getBridgeRetryAttempts(); + const retryDelayMs = getBridgeRetryDelayMs(); + let sessionRetried = false; + let lastError; + + for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + if (attempt > 0 && retryDelayMs > 0) { + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + } + + const targetClient = selectClientForTool(name, indexerClient, memoryClient); + if (!targetClient) { + throw new Error(`Tool ${name} not available on any configured MCP server`); + } + + try { + const result = await targetClient.callTool( { name, arguments: args, @@ -395,10 +478,33 @@ async function createBridgeServer(options) { undefined, { timeout: timeoutMs }, ); - return retryResult; + return result; + } catch (err) { + lastError = err; + + if (isSessionError(err) && !sessionRetried) { + debugLog( + "[ctxce] tools/call: detected remote MCP session error; reinitializing clients and retrying once: " + + String(err), + ); + await initializeRemoteClients(true); + sessionRetried = true; + continue; + } + + if (!isTransientToolError(err) || attempt === maxAttempts - 1) { + throw err; + } + + debugLog( + `[ctxce] tools/call: transient error (attempt ${attempt + 1}/${maxAttempts}), retrying: ` + + String(err), + ); + // Loop will retry } - throw err; } + + throw lastError || new Error("Unknown MCP tools/call error"); }); return server; From 3098828fddfa4507ec66780ce338376e5c1bf35b Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 00:27:17 +0000 Subject: [PATCH 19/55] feat: Introduce MCP bridge for auth and session management --- .env.example | 40 +++ ctx-mcp-bridge/README.md | 212 +++++++++++++ ctx-mcp-bridge/package.json | 2 +- ctx-mcp-bridge/src/authCli.js | 206 ++++++++++++ ctx-mcp-bridge/src/authConfig.js | 84 +++++ ctx-mcp-bridge/src/cli.js | 10 +- ctx-mcp-bridge/src/mcpServer.js | 51 ++- scripts/auth_backend.py | 297 ++++++++++++++++++ scripts/mcp_indexer_server.py | 24 ++ scripts/mcp_memory_server.py | 21 ++ scripts/remote_upload_client.py | 1 + scripts/upload_auth_utils.py | 47 +++ scripts/upload_service.py | 189 ++++++++++- .../context-engine-uploader/README.md | 80 ++++- .../context-engine-uploader/auth_utils.js | 234 ++++++++++++++ .../context-engine-uploader/extension.js | 29 +- .../context-engine-uploader/package.json | 7 +- 17 files changed, 1525 insertions(+), 9 deletions(-) create mode 100644 ctx-mcp-bridge/README.md create mode 100644 ctx-mcp-bridge/src/authCli.js create mode 100644 ctx-mcp-bridge/src/authConfig.js create mode 100644 scripts/auth_backend.py create mode 100644 scripts/upload_auth_utils.py create mode 100644 vscode-extension/context-engine-uploader/auth_utils.js diff --git a/.env.example b/.env.example index a53d3208..9ed72435 100644 --- a/.env.example +++ b/.env.example @@ -237,3 +237,43 @@ INFO_REQUEST_CONTEXT_LINES=5 # INFO_REQUEST_EXPLAIN_DEFAULT=0 # Enable relationship mapping by default (imports_from, calls, related_paths) # INFO_REQUEST_RELATIONSHIPS=0 + + +# --------------------------------------------------------------------------- +# Optional Auth & Bridge Configuration (OFF by default) +# --------------------------------------------------------------------------- + +# Global auth toggle for backend services (upload_service, MCP indexer/memory). +# When set to 1, services will require a valid session for protected operations. +# When unset or 0, auth is disabled and behavior matches previous versions. +# CTXCE_AUTH_ENABLED=0 + +# Shared token used by /auth/login for the bridge and other service clients. +# When CTXCE_AUTH_ENABLED=1 and this is set, /auth/login will only issue +# sessions when the provided token matches this value. +# CTXCE_AUTH_SHARED_TOKEN=change-me-dev-token + +# Optional admin token for creating additional users via /auth/users once the +# first user has been bootstrapped. If unset, only the initial user can be +# created (no additional users). +# CTXCE_AUTH_ADMIN_TOKEN=change-me-admin-token + +# Auth database location (default: sqlite file under WORK_DIR/.codebase). +# Use a SQLite URL for local/dev, or point to a different path. +# CTXCE_AUTH_DB_URL=sqlite:////work/.codebase/ctxce_auth.sqlite + +# Session TTL (seconds) for issued auth sessions. +# 0 or negative values disable expiry (sessions do not expire). When >0, +# active sessions are extended with a sliding window whenever they are used. +# CTXCE_AUTH_SESSION_TTL_SECONDS=0 + +# Bridge-side configuration (ctx-mcp-bridge): +# The bridge will POST to this URL for auth login and store the returned +# session id, then inject it into all MCP tool calls as the `session` field. +# CTXCE_AUTH_BACKEND_URL=http://localhost:8004 + +# Optional defaults for bridge CLI (env) auth: +# CTXCE_AUTH_TOKEN=dev-shared-token +# CTXCE_AUTH_USERNAME=you@example.com +# CTXCE_AUTH_PASSWORD=your-password + diff --git a/ctx-mcp-bridge/README.md b/ctx-mcp-bridge/README.md new file mode 100644 index 00000000..95a24bec --- /dev/null +++ b/ctx-mcp-bridge/README.md @@ -0,0 +1,212 @@ +# Context Engine MCP Bridge + +`@context-engine-bridge/context-engine-mcp-bridge` provides the `ctxce` CLI, a +Model Context Protocol (MCP) bridge that speaks to the Context Engine indexer +and memory servers and exposes them as a single MCP server. + +It is primarily used by the VS Code **Context Engine Uploader** extension, +available on the Marketplace: + +- + +The bridge can also be run standalone (e.g. from a terminal, or wired into +other MCP clients) as long as the Context Engine stack is running. + +## Prerequisites + +- Node.js **>= 18** (see `engines` in `package.json`). +- A running Context Engine stack (e.g. via `docker-compose.dev-remote.yml`) with: + - MCP indexer HTTP endpoint (default: `http://localhost:8003/mcp`). + - MCP memory HTTP endpoint (optional, default: `http://localhost:8002/mcp`). +- For optional auth: + - The upload/auth services must be configured with `CTXCE_AUTH_ENABLED=1` and + a reachable auth backend URL (e.g. `http://localhost:8004`). + +## Installation + +You can install the package globally, or run it via `npx`. + +### Global install + +```bash +npm install -g @context-engine-bridge/context-engine-mcp-bridge +``` + +This installs the `ctxce` (and `ctxce-bridge`) CLI in your PATH. + +### Using npx (no global install) + +```bash +npx @context-engine-bridge/context-engine-mcp-bridge ctxce --help +``` + +The examples below assume `ctxce` is available on your PATH; if you use `npx`, +just prefix commands with `npx @context-engine-bridge/context-engine-mcp-bridge`. + +## CLI overview + +The main entrypoint is: + +```bash +ctxce [...args] +``` + +Supported commands (from `src/cli.js`): + +- `ctxce mcp-serve` – stdio MCP bridge (for stdio-based MCP clients). +- `ctxce mcp-http-serve` – HTTP MCP bridge (for HTTP-based MCP clients). +- `ctxce auth ` – auth helper commands (`login`, `status`, `logout`). + +### Environment variables + +These environment variables are respected by the bridge: + +- `CTXCE_INDEXER_URL` – MCP indexer URL (default: `http://localhost:8003/mcp`). +- `CTXCE_MEMORY_URL` – MCP memory URL, or empty/omitted to disable memory + (default: `http://localhost:8002/mcp`). +- `CTXCE_HTTP_PORT` – port for `mcp-http-serve` (default: `30810`). + +For auth (optional, shared with the upload/auth backend): + +- `CTXCE_AUTH_ENABLED` – whether auth is enabled in the backend. +- `CTXCE_AUTH_BACKEND_URL` – auth backend URL (e.g. `http://localhost:8004`). +- `CTXCE_AUTH_TOKEN` – dev/shared token for `ctxce auth login`. +- `CTXCE_AUTH_SESSION_TTL_SECONDS` – session TTL / sliding expiry (seconds). + +The CLI also stores auth sessions in `~/.ctxce/auth.json`, keyed by backend URL. + +## Running the MCP bridge (stdio) + +The stdio bridge is suitable for MCP clients that speak stdio directly (for +example, certain editors or tools that expect an MCP server on stdin/stdout). + +```bash +ctxce mcp-serve \ + --workspace /path/to/your/workspace \ + --indexer-url http://localhost:8003/mcp \ + --memory-url http://localhost:8002/mcp +``` + +Flags: + +- `--workspace` / `--path` – workspace root (default: current working directory). +- `--indexer-url` – override indexer URL (default: `CTXCE_INDEXER_URL` or + `http://localhost:8003/mcp`). +- `--memory-url` – override memory URL (default: `CTXCE_MEMORY_URL` or + disabled when empty). + +## Running the MCP bridge (HTTP) + +The HTTP bridge exposes the MCP server via an HTTP endpoint (default +`http://127.0.0.1:30810/mcp`) and is what the VS Code extension uses in its +`http` transport mode. + +```bash +ctxce mcp-http-serve \ + --workspace /path/to/your/workspace \ + --indexer-url http://localhost:8003/mcp \ + --memory-url http://localhost:8002/mcp \ + --port 30810 +``` + +Flags: + +- `--workspace` / `--path` – workspace root (default: current working directory). +- `--indexer-url` – MCP indexer URL. +- `--memory-url` – MCP memory URL (or omit/empty to disable memory). +- `--port` – HTTP port for the bridge (default: `CTXCE_HTTP_PORT` + or `30810`). + +Once running, you can point an MCP client at: + +```text +http://127.0.0.1:/mcp +``` + +## Auth helper commands (`ctxce auth ...`) + +These commands are used both by the VS Code extension and standalone flows to +log in and manage auth sessions for the backend. + +### Login (token) + +```bash +ctxce auth login \ + --backend-url http://localhost:8004 \ + --token $CTXCE_AUTH_SHARED_TOKEN +``` + +This hits the backend `/auth/login` endpoint and stores a session entry in +`~/.ctxce/auth.json` under the given backend URL. + +### Login (username/password) + +```bash +ctxce auth login \ + --backend-url http://localhost:8004 \ + --username your-user \ + --password your-password +``` + +This calls `/auth/login/password` and persists the returned session the same +way as the token flow. + +### Status + +Human-readable status: + +```bash +ctxce auth status --backend-url http://localhost:8004 +``` + +Machine-readable status (used by the VS Code extension): + +```bash +ctxce auth status --backend-url http://localhost:8004 --json +``` + +The `--json` variant prints a single JSON object to stdout, for example: + +```json +{ + "backendUrl": "http://localhost:8004", + "state": "ok", // "ok" | "missing" | "expired" | "missing_backend" + "sessionId": "...", + "userId": "user-123", + "expiresAt": 0 // 0 or a Unix timestamp +} +``` + +Exit codes: + +- `0` – `state: "ok"` (valid session present). +- `1` – `state: "missing"` or `"missing_backend"`. +- `2` – `state: "expired"`. + +### Logout + +```bash +ctxce auth logout --backend-url http://localhost:8004 +``` + +Removes the stored auth entry for the given backend URL from +`~/.ctxce/auth.json`. + +## Relationship to the VS Code extension + +The VS Code **Context Engine Uploader** extension is the recommended way to use +this bridge for day-to-day development. It: + +- Launches the standalone upload client to push code into the remote stack. +- Starts/stops the MCP HTTP bridge (`ctxce mcp-http-serve`) for the active + workspace when `autoStartMcpBridge` is enabled. +- Uses `ctxce auth status --json` and `ctxce auth login` under the hood to + manage user sessions via UI prompts. + +This package README is aimed at advanced users who want to: + +- Run the MCP bridge outside of VS Code. +- Integrate the Context Engine MCP servers with other MCP-compatible clients. + +You can safely mix both approaches: the extension and the standalone bridge +share the same auth/session storage in `~/.ctxce/auth.json`. diff --git a/ctx-mcp-bridge/package.json b/ctx-mcp-bridge/package.json index 3bcb4244..0c971840 100644 --- a/ctx-mcp-bridge/package.json +++ b/ctx-mcp-bridge/package.json @@ -1,6 +1,6 @@ { "name": "@context-engine-bridge/context-engine-mcp-bridge", - "version": "0.0.5", + "version": "0.0.6", "description": "Context Engine MCP bridge (http/stdio proxy combining indexer + memory servers)", "bin": { "ctxce": "bin/ctxce.js", diff --git a/ctx-mcp-bridge/src/authCli.js b/ctx-mcp-bridge/src/authCli.js new file mode 100644 index 00000000..eadb0bee --- /dev/null +++ b/ctx-mcp-bridge/src/authCli.js @@ -0,0 +1,206 @@ +import process from "node:process"; +import { loadAuthEntry, saveAuthEntry, deleteAuthEntry } from "./authConfig.js"; + +function parseAuthArgs(args) { + let backendUrl = process.env.CTXCE_AUTH_BACKEND_URL || ""; + let token = process.env.CTXCE_AUTH_TOKEN || ""; + let username = process.env.CTXCE_AUTH_USERNAME || ""; + let password = process.env.CTXCE_AUTH_PASSWORD || ""; + let outputJson = false; + for (let i = 0; i < args.length; i += 1) { + const a = args[i]; + if ((a === "--backend-url" || a === "--auth-url") && i + 1 < args.length) { + backendUrl = args[i + 1]; + i += 1; + continue; + } + if ((a === "--token" || a === "--api-key") && i + 1 < args.length) { + token = args[i + 1]; + i += 1; + continue; + } + if ((a === "--username" || a === "--user") && i + 1 < args.length) { + username = args[i + 1]; + i += 1; + continue; + } + if ((a === "--password" || a === "--pass") && i + 1 < args.length) { + password = args[i + 1]; + i += 1; + continue; + } + if (a === "--json" || a === "-j") { + outputJson = true; + continue; + } + } + return { backendUrl, token, username, password, outputJson }; +} + +function getBackendUrl(backendUrl) { + return (backendUrl || process.env.CTXCE_AUTH_BACKEND_URL || "").trim(); +} + +function requireBackendUrl(backendUrl) { + const url = getBackendUrl(backendUrl); + if (!url) { + console.error("[ctxce] Auth backend URL not configured. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + process.exit(1); + } + return url; +} + +function outputJsonStatus(url, state, entry, rawExpires) { + const expiresAt = typeof rawExpires === "number" + ? rawExpires + : entry && typeof entry.expiresAt === "number" + ? entry.expiresAt + : null; + console.log(JSON.stringify({ + backendUrl: url, + state, + sessionId: entry && entry.sessionId ? entry.sessionId : null, + userId: entry && entry.userId ? entry.userId : null, + expiresAt, + })); +} + +async function doLogin(args) { + const { backendUrl, token, username, password } = parseAuthArgs(args); + const url = requireBackendUrl(backendUrl); + const trimmedUser = (username || "").trim(); + const usePassword = trimmedUser && (password || "").length > 0; + + let body; + let target; + if (usePassword) { + body = { + username: trimmedUser, + password, + workspace: process.cwd(), + }; + target = url.replace(/\/+$/, "") + "/auth/login/password"; + } else { + body = { + client: "ctxce", + workspace: process.cwd(), + }; + if (token) { + body.token = token; + } + target = url.replace(/\/+$/, "") + "/auth/login"; + } + let resp; + try { + resp = await fetch(target, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + } catch (err) { + console.error("[ctxce] Auth login request failed:", String(err)); + process.exit(1); + } + if (!resp || !resp.ok) { + console.error("[ctxce] Auth login failed with status", resp ? resp.status : ""); + process.exit(1); + } + let data; + try { + data = await resp.json(); + } catch (err) { + data = {}; + } + const sessionId = data.session_id || data.sessionId || null; + const userId = data.user_id || data.userId || null; + const expiresAt = data.expires_at || data.expiresAt || null; + if (!sessionId) { + console.error("[ctxce] Auth login response missing session id."); + process.exit(1); + } + saveAuthEntry(url, { sessionId, userId, expiresAt }); + console.error("[ctxce] Auth login successful for", url); +} + +async function doStatus(args) { + const { backendUrl, outputJson } = parseAuthArgs(args); + const url = getBackendUrl(backendUrl); + if (!url) { + if (outputJson) { + outputJsonStatus("", "missing_backend", null, null); + process.exit(1); + } + console.error("[ctxce] Auth backend URL not configured. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + process.exit(1); + } + let entry; + try { + entry = loadAuthEntry(url); + } catch (err) { + entry = null; + } + const nowSecs = Math.floor(Date.now() / 1000); + const rawExpires = entry && typeof entry.expiresAt === "number" ? entry.expiresAt : null; + const hasSession = !!(entry && typeof entry.sessionId === "string" && entry.sessionId); + const expired = !!(rawExpires && rawExpires > 0 && rawExpires < nowSecs); + + if (!entry || !hasSession) { + if (outputJson) { + outputJsonStatus(url, "missing", null, rawExpires); + process.exit(1); + } + console.error("[ctxce] Not logged in for", url); + process.exit(1); + } + + if (expired) { + if (outputJson) { + outputJsonStatus(url, "expired", entry, rawExpires); + process.exit(2); + } + console.error("[ctxce] Stored auth session appears expired for", url); + if (rawExpires) { + console.error("[ctxce] Session expired at", rawExpires); + } + process.exit(2); + } + + if (outputJson) { + outputJsonStatus(url, "ok", entry, rawExpires); + return; + } + console.error("[ctxce] Logged in to", url, "as", entry.userId || ""); + if (rawExpires) { + console.error("[ctxce] Session expires at", rawExpires); + } +} + +async function doLogout(args) { + const { backendUrl } = parseAuthArgs(args); + const url = requireBackendUrl(backendUrl); + const entry = loadAuthEntry(url); + if (!entry) { + console.error("[ctxce] No stored auth session for", url); + return; + } + deleteAuthEntry(url); + console.error("[ctxce] Logged out from", url); +} + +export async function runAuthCommand(subcommand, args) { + const sub = (subcommand || "").toLowerCase(); + if (sub === "login") { + await doLogin(args || []); + return; + } + if (sub === "status") { + await doStatus(args || []); + return; + } + if (sub === "logout") { + await doLogout(args || []); + return; + } + console.error("Usage: ctxce auth [--backend-url ] [--token ]"); + process.exit(1); +} diff --git a/ctx-mcp-bridge/src/authConfig.js b/ctx-mcp-bridge/src/authConfig.js new file mode 100644 index 00000000..7cd63bb1 --- /dev/null +++ b/ctx-mcp-bridge/src/authConfig.js @@ -0,0 +1,84 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const CONFIG_DIR_NAME = ".ctxce"; +const CONFIG_BASENAME = "auth.json"; + +function getConfigPath() { + const home = os.homedir() || process.cwd(); + const dir = path.join(home, CONFIG_DIR_NAME); + return path.join(dir, CONFIG_BASENAME); +} + +function readConfig() { + try { + const cfgPath = getConfigPath(); + const raw = fs.readFileSync(cfgPath, "utf8"); + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === "object") { + return parsed; + } + } catch (err) { + } + return {}; +} + +function writeConfig(data) { + try { + const cfgPath = getConfigPath(); + const dir = path.dirname(cfgPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + fs.writeFileSync(cfgPath, JSON.stringify(data, null, 2), "utf8"); + } catch (err) { + } +} + +export function loadAuthEntry(backendUrl) { + if (!backendUrl) { + return null; + } + const all = readConfig(); + const key = String(backendUrl); + const entry = all[key]; + if (!entry || typeof entry !== "object") { + return null; + } + return entry; +} + +export function saveAuthEntry(backendUrl, entry) { + if (!backendUrl || !entry || typeof entry !== "object") { + return; + } + const all = readConfig(); + const key = String(backendUrl); + all[key] = entry; + writeConfig(all); +} + +export function deleteAuthEntry(backendUrl) { + if (!backendUrl) { + return; + } + const all = readConfig(); + const key = String(backendUrl); + if (Object.prototype.hasOwnProperty.call(all, key)) { + delete all[key]; + writeConfig(all); + } +} + +export function loadAnyAuthEntry() { + const all = readConfig(); + const keys = Object.keys(all); + for (const key of keys) { + const entry = all[key]; + if (entry && typeof entry === "object") { + return { backendUrl: key, entry }; + } + } + return null; +} diff --git a/ctx-mcp-bridge/src/cli.js b/ctx-mcp-bridge/src/cli.js index 3df486d8..4d995645 100644 --- a/ctx-mcp-bridge/src/cli.js +++ b/ctx-mcp-bridge/src/cli.js @@ -4,11 +4,19 @@ import process from "node:process"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { runMcpServer, runHttpMcpServer } from "./mcpServer.js"; +import { runAuthCommand } from "./authCli.js"; export async function runCli() { const argv = process.argv.slice(2); const cmd = argv[0]; + if (cmd === "auth") { + const sub = argv[1] || ""; + const args = argv.slice(2); + await runAuthCommand(sub, args); + return; + } + if (cmd === "mcp-http-serve") { const args = argv.slice(1); let workspace = process.cwd(); @@ -109,7 +117,7 @@ export async function runCli() { // eslint-disable-next-line no-console console.error( - `Usage: ${binName} mcp-serve [--workspace ] [--indexer-url ] [--memory-url ] | ${binName} mcp-http-serve [--workspace ] [--indexer-url ] [--memory-url ] [--port ]`, + `Usage: ${binName} mcp-serve [--workspace ] [--indexer-url ] [--memory-url ] | ${binName} mcp-http-serve [--workspace ] [--indexer-url ] [--memory-url ] [--port ] | ${binName} auth [--backend-url ] [--token ] [--username --password ]`, ); process.exit(1); } diff --git a/ctx-mcp-bridge/src/mcpServer.js b/ctx-mcp-bridge/src/mcpServer.js index f967f83a..fa2a5719 100644 --- a/ctx-mcp-bridge/src/mcpServer.js +++ b/ctx-mcp-bridge/src/mcpServer.js @@ -247,6 +247,7 @@ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/ import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js"; +import { loadAnyAuthEntry, loadAuthEntry } from "./authConfig.js"; async function createBridgeServer(options) { const workspace = options.workspace || process.cwd(); @@ -281,8 +282,54 @@ async function createBridgeServer(options) { // future this can be made user-aware (e.g. from auth), but for now we // keep it deterministic per workspace to help the indexer reuse // session-scoped defaults. - const sessionId = - process.env.CTXCE_SESSION_ID || `ctxce-${Buffer.from(workspace).toString("hex").slice(0, 24)}`; + const explicitSession = process.env.CTXCE_SESSION_ID || ""; + const authBackendUrl = process.env.CTXCE_AUTH_BACKEND_URL || ""; + let sessionId = explicitSession; + + if (!sessionId) { + let backendToUse = authBackendUrl; + let entry = null; + + if (backendToUse) { + try { + entry = loadAuthEntry(backendToUse); + } catch (err) { + entry = null; + } + } + + if (!entry) { + try { + const any = loadAnyAuthEntry(); + if (any && any.entry) { + backendToUse = any.backendUrl; + entry = any.entry; + } + } catch (err) { + entry = null; + } + } + + if (entry) { + let expired = false; + const rawExpires = entry.expiresAt; + if (typeof rawExpires === "number" && Number.isFinite(rawExpires) && rawExpires > 0) { + const nowSecs = Math.floor(Date.now() / 1000); + if (rawExpires < nowSecs) { + expired = true; + } + } + if (!expired && typeof entry.sessionId === "string" && entry.sessionId) { + sessionId = entry.sessionId; + } else if (expired) { + debugLog("[ctxce] Stored auth session appears expired; please run `ctxce auth login` again."); + } + } + } + + if (!sessionId) { + sessionId = `ctxce-${Buffer.from(workspace).toString("hex").slice(0, 24)}`; + } // Best-effort: inform the indexer of default collection and session. // If this fails we still proceed, falling back to per-call injection. diff --git a/scripts/auth_backend.py b/scripts/auth_backend.py new file mode 100644 index 00000000..52750ebe --- /dev/null +++ b/scripts/auth_backend.py @@ -0,0 +1,297 @@ +"""Authentication backend for Context-Engine services. + +Provides a minimal, SQLite-backed user and session store with +password hashing and optional shared-token based session issuance. + +Design notes (PoC-friendly, forward compatible): + +- Storage schema is intentionally simple and portable (TEXT/INTEGER fields + only) so it can be migrated to a real RDBMS (Postgres/MySQL) later without + changing the logical model. +- AUTH_DB_URL accepts a SQLite-style URL today, but callers should treat it + as an abstract database URL; future versions may use Alembic-style schema + migrations and support multiple engines. +- Current focus is users + sessions only. In a fuller deployment, this module + is the natural place to grow organization and collection metadata, including + mapping users/orgs to existing Qdrant collections and enforcing collection- + level ACLs. + +Auth is fully opt-in via environment variables and can be reused by +multiple services (upload, dedicated auth service, MCP indexers, etc.). +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sqlite3 +import uuid +from contextlib import contextmanager +from datetime import datetime +from typing import Any, Dict, Optional + +# Configuration +WORK_DIR = os.environ.get("WORK_DIR", "/work") +AUTH_ENABLED = ( + str(os.environ.get("CTXCE_AUTH_ENABLED", "0")) + .strip() + .lower() + in {"1", "true", "yes", "on"} +) +_default_auth_db_path = os.path.join(WORK_DIR, ".codebase", "ctxce_auth.sqlite") +AUTH_DB_URL = os.environ.get("CTXCE_AUTH_DB_URL") or f"sqlite:///{_default_auth_db_path}" +AUTH_SHARED_TOKEN = os.environ.get("CTXCE_AUTH_SHARED_TOKEN") + +_SESSION_TTL_SECONDS_DEFAULT = 0 +try: + _raw_ttl = os.environ.get("CTXCE_AUTH_SESSION_TTL_SECONDS") + if _raw_ttl is not None and str(_raw_ttl).strip() != "": + AUTH_SESSION_TTL_SECONDS = int(str(_raw_ttl).strip()) + else: + AUTH_SESSION_TTL_SECONDS = _SESSION_TTL_SECONDS_DEFAULT +except Exception: + AUTH_SESSION_TTL_SECONDS = _SESSION_TTL_SECONDS_DEFAULT + + +class AuthDisabledError(Exception): + """Raised when auth is disabled via configuration.""" + + +class AuthInvalidToken(Exception): + """Raised when a shared token login attempt fails validation.""" + + +def _get_auth_db_path() -> str: + raw = AUTH_DB_URL or "" + if raw.startswith("sqlite///"): + return raw[len("sqlite///") :] + if raw.startswith("sqlite://"): + return raw[len("sqlite://") :] + return raw + + +@contextmanager +def _db_connection(): + path = _get_auth_db_path() + conn = sqlite3.connect(path) + try: + yield conn + finally: + conn.close() + + +def _ensure_auth_db() -> None: + path = _get_auth_db_path() + if not path: + return + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + except Exception: + pass + with _db_connection() as conn: + with conn: + conn.execute( + "CREATE TABLE IF NOT EXISTS sessions (id TEXT PRIMARY KEY, user_id TEXT, created_at INTEGER, expires_at INTEGER, metadata_json TEXT)" + ) + conn.execute( + "CREATE TABLE IF NOT EXISTS users (id TEXT PRIMARY KEY, username TEXT UNIQUE NOT NULL, password_hash TEXT NOT NULL, created_at INTEGER NOT NULL, metadata_json TEXT)" + ) + + +def _hash_password(password: str) -> str: + if not isinstance(password, str) or not password: + raise ValueError("Password is required") + salt = os.urandom(16) + iterations = 200_000 + dk = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations) + return f"pbkdf2_sha256${iterations}${salt.hex()}${dk.hex()}" + + +def _verify_password(password: str, encoded: str) -> bool: + try: + scheme, iter_s, salt_hex, hash_hex = encoded.split("$", 3) + if scheme != "pbkdf2_sha256": + return False + iterations = int(iter_s) + salt = bytes.fromhex(salt_hex) + dk = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations) + return dk.hex() == hash_hex + except Exception: + return False + + +def create_user( + username: str, password: str, metadata: Optional[Dict[str, Any]] = None +) -> Dict[str, Any]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_auth_db() + path = _get_auth_db_path() + now_ts = int(datetime.now().timestamp()) + password_hash = _hash_password(password) + meta_json: Optional[str] = None + if metadata: + try: + meta_json = json.dumps(metadata) + except Exception: + meta_json = None + user_id = uuid.uuid4().hex + with _db_connection() as conn: + with conn: + conn.execute( + "INSERT INTO users (id, username, password_hash, created_at, metadata_json) VALUES (?, ?, ?, ?, ?)", + (user_id, username, password_hash, now_ts, meta_json), + ) + return {"user_id": user_id, "username": username} + + +def _get_user_by_username(username: str) -> Optional[Dict[str, Any]]: + _ensure_auth_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute( + "SELECT id, username, password_hash, created_at, metadata_json FROM users WHERE username = ?", + (username,), + ) + row = cur.fetchone() + if not row: + return None + return { + "id": row[0], + "username": row[1], + "password_hash": row[2], + "created_at": row[3], + "metadata_json": row[4], + } + + +def has_any_users() -> bool: + """Return True if at least one user exists. + + Used by HTTP layers to allow first-user bootstrap flows when the + database is empty. Raises AuthDisabledError when auth is disabled. + """ + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_auth_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute("SELECT 1 FROM users LIMIT 1") + row = cur.fetchone() + return bool(row) + + +def authenticate_user(username: str, password: str) -> Optional[Dict[str, Any]]: + user = _get_user_by_username(username) + if not user: + return None + if not _verify_password(password, user.get("password_hash") or ""): + return None + return user + + +def create_session( + user_id: str, + metadata: Optional[Dict[str, Any]] = None, + ttl_seconds: int = AUTH_SESSION_TTL_SECONDS, +) -> Dict[str, Any]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_auth_db() + path = _get_auth_db_path() + now_ts = int(datetime.now().timestamp()) + ttl_val = int(ttl_seconds or 0) + if ttl_val <= 0: + expires_ts = 0 + else: + expires_ts = now_ts + ttl_val + meta_json: Optional[str] = None + if metadata: + try: + meta_json = json.dumps(metadata) + except Exception: + meta_json = None + session_id = uuid.uuid4().hex + with _db_connection() as conn: + with conn: + conn.execute( + "INSERT OR REPLACE INTO sessions (id, user_id, created_at, expires_at, metadata_json) VALUES (?, ?, ?, ?, ?)", + (session_id, user_id, now_ts, expires_ts, meta_json), + ) + return {"session_id": session_id, "user_id": user_id, "expires_at": expires_ts} + + +def create_session_for_token( + client: str, + workspace: Optional[str] = None, + token: Optional[str] = None, +) -> Dict[str, Any]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + if AUTH_SHARED_TOKEN: + if not token or token != AUTH_SHARED_TOKEN: + raise AuthInvalidToken("Invalid auth token") + user_id = client or "ctxce" + meta: Dict[str, Any] = {} + if workspace: + meta["workspace"] = workspace + return create_session(user_id=user_id, metadata=meta) + + +def validate_session(session_id: str) -> Optional[Dict[str, Any]]: + """Validate a session id and return its record if active. + + Returns a dict with keys {id, user_id, created_at, expires_at, metadata} + when valid, or None when missing/expired/unknown. Raises AuthDisabledError + when auth is disabled. + """ + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + sid = (session_id or "").strip() + if not sid: + return None + _ensure_auth_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute( + "SELECT id, user_id, created_at, expires_at, metadata_json FROM sessions WHERE id = ?", + (sid,), + ) + row = cur.fetchone() + if not row: + return None + now_ts = int(datetime.now().timestamp()) + expires_ts = int(row[3] or 0) + if expires_ts and expires_ts < now_ts: + return None + if AUTH_SESSION_TTL_SECONDS > 0 and expires_ts: + remaining = expires_ts - now_ts + if remaining < AUTH_SESSION_TTL_SECONDS // 2: + new_expires_ts = now_ts + AUTH_SESSION_TTL_SECONDS + try: + with _db_connection() as conn2: + with conn2: + conn2.execute( + "UPDATE sessions SET expires_at = ? WHERE id = ?", + (new_expires_ts, sid), + ) + expires_ts = new_expires_ts + except Exception: + pass + meta: Optional[Dict[str, Any]] = None + raw_meta = row[4] + if isinstance(raw_meta, str) and raw_meta.strip(): + try: + obj = json.loads(raw_meta) + if isinstance(obj, dict): + meta = obj + except Exception: + meta = None + return { + "id": row[0], + "user_id": row[1], + "created_at": int(row[2] or 0), + "expires_at": expires_ts, + "metadata": meta or {}, + } diff --git a/scripts/mcp_indexer_server.py b/scripts/mcp_indexer_server.py index 1b2852f7..50048e06 100644 --- a/scripts/mcp_indexer_server.py +++ b/scripts/mcp_indexer_server.py @@ -123,6 +123,27 @@ def safe_bool(value, default=False, logger=None, context=""): return default +try: + from scripts.auth_backend import AUTH_ENABLED as AUTH_ENABLED_AUTH, validate_session as _auth_validate_session +except Exception: + AUTH_ENABLED_AUTH = False + + def _auth_validate_session(session_id: str): # type: ignore[no-redef] + return None + + +def _require_auth_session(session: Optional[str]) -> Optional[Dict[str, Any]]: + if not AUTH_ENABLED_AUTH: + return None + sid = (session or "").strip() + if not sid: + raise ValidationError("Missing session for authorized operation") + info = _auth_validate_session(sid) + if not info: + raise ValidationError("Invalid or expired session") + return info + + # Global lock to guard temporary env toggles used during ReFRAG retrieval/decoding _ENV_LOCK = threading.Lock() @@ -1812,6 +1833,9 @@ async def repo_search( - path_glob=["scripts/**","**/*.py"], language="python" - symbol="context_answer", under="scripts" """ + # Enforce auth when enabled (no-op when CTXCE_AUTH_ENABLED is false) + _require_auth_session(session) + # Handle queries alias (explicit parameter) if queries is not None and (query is None or (isinstance(query, str) and str(query).strip() == "")): query = queries diff --git a/scripts/mcp_memory_server.py b/scripts/mcp_memory_server.py index 6644fe07..446935bf 100644 --- a/scripts/mcp_memory_server.py +++ b/scripts/mcp_memory_server.py @@ -152,6 +152,26 @@ def _inner(fn): _SESSION_CTX_LOCK = threading.Lock() SESSION_DEFAULTS_BY_SESSION: "WeakKeyDictionary[Any, Dict[str, Any]]" = WeakKeyDictionary() +try: + from scripts.auth_backend import AUTH_ENABLED as AUTH_ENABLED_AUTH, validate_session as _auth_validate_session +except Exception: + AUTH_ENABLED_AUTH = False + + def _auth_validate_session(session_id: str): # type: ignore[no-redef] + return None + + +def _require_auth_session(session: Optional[str]) -> Optional[Dict[str, Any]]: + if not AUTH_ENABLED_AUTH: + return None + sid = (session or "").strip() + if not sid: + raise Exception("Missing session for authorized operation") + info = _auth_validate_session(sid) + if not info: + raise Exception("Invalid or expired session") + return info + def _start_readyz_server(): try: @@ -354,6 +374,7 @@ def store( First call may be slower because the embedding model loads lazily. """ + _require_auth_session(session) coll = _resolve_collection(collection, session=session, ctx=ctx, extra_kwargs=kwargs) _ensure_once(coll) model = _get_embedding_model() diff --git a/scripts/remote_upload_client.py b/scripts/remote_upload_client.py index b6bd54dd..37646e58 100644 --- a/scripts/remote_upload_client.py +++ b/scripts/remote_upload_client.py @@ -29,6 +29,7 @@ import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry +from scripts.upload_auth_utils import get_auth_session # Configure logging logging.basicConfig(level=logging.INFO) diff --git a/scripts/upload_auth_utils.py b/scripts/upload_auth_utils.py new file mode 100644 index 00000000..76b15e80 --- /dev/null +++ b/scripts/upload_auth_utils.py @@ -0,0 +1,47 @@ +import os +import json +import time +from typing import Any + + +def get_auth_session(upload_endpoint: str) -> str: + """Resolve auth session from environment or ~/.ctxce/auth.json. + + This mirrors the existing behavior used by the upload clients: + - Prefer CTXCE_UPLOAD_SESSION_ID / CTXCE_SESSION_ID from the environment. + - Fall back to ~/.ctxce/auth.json keyed by the upload endpoint (with and without + a trailing slash), honoring an optional numeric expiresAt/expires_at field. + - Treat expiresAt <= 0 or missing as non-expiring. + + Returns an empty string when no usable session is found. + """ + try: + sess = (os.environ.get("CTXCE_UPLOAD_SESSION_ID") or os.environ.get("CTXCE_SESSION_ID") or "").strip() + except Exception: + sess = "" + if sess: + return sess + + try: + home = os.path.expanduser("~") + cfg_path = os.path.join(home, ".ctxce", "auth.json") + if not os.path.exists(cfg_path): + return "" + with open(cfg_path, "r", encoding="utf-8") as f: + raw: Any = json.load(f) + if not isinstance(raw, dict): + return "" + key = upload_endpoint.rstrip("/") + entry = raw.get(key) or raw.get(upload_endpoint) + if not isinstance(entry, dict): + return "" + sid = entry.get("sessionId") or entry.get("session_id") + exp = entry.get("expiresAt") or entry.get("expires_at") + now_secs = int(time.time()) + if isinstance(exp, (int, float)) and exp > 0: + if exp >= now_secs: + return (sid or "").strip() + return "" + return (sid or "").strip() + except Exception: + return "" diff --git a/scripts/upload_service.py b/scripts/upload_service.py index 4807ef0f..8de5bfed 100644 --- a/scripts/upload_service.py +++ b/scripts/upload_service.py @@ -22,6 +22,18 @@ from fastapi.responses import JSONResponse from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel, Field +from scripts.auth_backend import ( + AuthDisabledError, + AuthInvalidToken, + authenticate_user, + create_session, + create_session_for_token, + create_user, + has_any_users, + validate_session, + AUTH_ENABLED, + AUTH_SESSION_TTL_SECONDS, +) # Import existing workspace state and indexing functions try: @@ -111,6 +123,40 @@ class HealthResponse(BaseModel): qdrant_url: str work_dir: str + +class AuthLoginRequest(BaseModel): + client: str + workspace: Optional[str] = None + token: Optional[str] = None + + +class AuthLoginResponse(BaseModel): + session_id: str + user_id: Optional[str] = None + expires_at: Optional[int] = None + + +class AuthStatusResponse(BaseModel): + enabled: bool + has_users: Optional[bool] = None + session_ttl_seconds: int + + +class AuthUserCreateRequest(BaseModel): + username: str + password: str + + +class AuthUserCreateResponse(BaseModel): + user_id: str + username: str + + +class PasswordLoginRequest(BaseModel): + username: str + password: str + workspace: Optional[str] = None + def get_workspace_key(workspace_path: str) -> str: """Generate 16-char hash for collision avoidance in remote uploads. @@ -193,7 +239,6 @@ def _cleanup_empty_dirs(path: Path, stop_at: Path) -> None: except Exception: break - def process_delta_bundle(workspace_path: str, bundle_path: Path, manifest: Dict[str, Any]) -> Dict[str, int]: """Process delta bundle and return operation counts.""" operations_count = { @@ -410,6 +455,128 @@ async def _process_bundle_background( pass +@app.get("/auth/status", response_model=AuthStatusResponse) +async def auth_status(): + try: + if not AUTH_ENABLED: + return AuthStatusResponse( + enabled=False, + has_users=None, + session_ttl_seconds=AUTH_SESSION_TTL_SECONDS, + ) + try: + users_exist = has_any_users() + except AuthDisabledError: + return AuthStatusResponse( + enabled=False, + has_users=None, + session_ttl_seconds=AUTH_SESSION_TTL_SECONDS, + ) + return AuthStatusResponse( + enabled=True, + has_users=users_exist, + session_ttl_seconds=AUTH_SESSION_TTL_SECONDS, + ) + except Exception as e: + logger.error(f"[upload_service] Failed to report auth status: {e}") + raise HTTPException(status_code=500, detail="Failed to read auth status") + + +@app.post("/auth/login", response_model=AuthLoginResponse) +async def auth_login(payload: AuthLoginRequest): + try: + session = create_session_for_token( + client=payload.client, + workspace=payload.workspace, + token=payload.token, + ) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except AuthInvalidToken: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid auth token") + except Exception as e: + logger.error(f"[upload_service] Failed to create auth session: {e}") + raise HTTPException(status_code=500, detail="Failed to create auth session") + return AuthLoginResponse( + session_id=session.get("session_id"), + user_id=session.get("user_id"), + expires_at=session.get("expires_at"), + ) + + +@app.post("/auth/users", response_model=AuthUserCreateResponse) +async def auth_create_user(payload: AuthUserCreateRequest, request: Request): + try: + first_user = not has_any_users() + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to check user state: {e}") + raise HTTPException(status_code=500, detail="Failed to inspect user state") + + admin_token = os.environ.get("CTXCE_AUTH_ADMIN_TOKEN") or os.environ.get("CTXCE_AUTH_SHARED_TOKEN") + if not first_user: + if not admin_token: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin token not configured", + ) + header = request.headers.get("X-Admin-Token") + if not header or header != admin_token: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid admin token", + ) + + try: + user = create_user(payload.username, payload.password) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to create user: {e}") + msg = str(e) + if "UNIQUE" in msg or "unique" in msg: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Username already exists", + ) + raise HTTPException(status_code=500, detail="Failed to create user") + + return AuthUserCreateResponse(user_id=user.get("user_id"), username=user.get("username")) + + +@app.post("/auth/login/password", response_model=AuthLoginResponse) +async def auth_login_password(payload: PasswordLoginRequest): + try: + user = authenticate_user(payload.username, payload.password) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Error authenticating user: {e}") + raise HTTPException(status_code=500, detail="Authentication error") + + if not user: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid credentials") + + meta: Optional[Dict[str, Any]] = None + if payload.workspace: + meta = {"workspace": payload.workspace} + + try: + session = create_session(user_id=user.get("id"), metadata=meta) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to create session for user: {e}") + raise HTTPException(status_code=500, detail="Failed to create auth session") + + return AuthLoginResponse( + session_id=session.get("session_id"), + user_id=session.get("user_id"), + expires_at=session.get("expires_at"), + ) + + @app.get("/health", response_model=HealthResponse) async def health_check(): """Health check endpoint.""" @@ -465,6 +632,7 @@ async def upload_delta_bundle( force: Optional[bool] = Form(False), source_path: Optional[str] = Form(None), logical_repo_id: Optional[str] = Form(None), + session: Optional[str] = Form(None), ): """Upload and process delta bundle.""" start_time = datetime.now() @@ -472,6 +640,25 @@ async def upload_delta_bundle( try: logger.info(f"[upload_service] Begin processing upload for workspace={workspace_path} from {client_host}") + + if AUTH_ENABLED: + session_value = (session or "").strip() + try: + record = validate_session(session_value) + except AuthDisabledError: + record = None + except Exception as e: + logger.error(f"[upload_service] Failed to validate auth session for upload: {e}") + raise HTTPException( + status_code=500, + detail="Failed to validate auth session", + ) + if record is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired session", + ) + # Validate workspace path workspace = Path(workspace_path) if not workspace.is_absolute(): diff --git a/vscode-extension/context-engine-uploader/README.md b/vscode-extension/context-engine-uploader/README.md index 89138292..a0d7f02b 100644 --- a/vscode-extension/context-engine-uploader/README.md +++ b/vscode-extension/context-engine-uploader/README.md @@ -1,6 +1,10 @@ Context Engine Uploader ======================= +Install +------- +- Install from the VS Code Marketplace (search for "Context Engine Uploader" or publisher `context-engine`). You can also install it directly from the Extensions view in VS Code. + Features -------- - Runs a force sync (`Index Codebase`) followed by watch mode to keep a remote Context Engine instance in sync with your workspace. @@ -52,7 +56,7 @@ MCP bridge (ctx-mcp-bridge) & MCP config lifecycle - Why use the bridge instead of two direct MCP entries? - **Single server entry:** IDEs only need to register one MCP server (`context-engine`) instead of juggling separate `qdrant-indexer` and `memory` entries, avoiding coordination mistakes. - **Shared session defaults:** the bridge loads `ctx_config.json` and injects collection name, repo metadata, and any other ctx defaults so every IDE window talks to the right collection without hand-editing `.mcp.json`. - - **Per-user credential isolation:** each IDE maintains its own MCP session while the bridge multiplexes upstream calls, so user preferences (future auth) remain per client even though the backend pair is shared. + - **Per-user credential isolation:** each IDE maintains its own MCP session while the bridge multiplexes upstream calls. When you enable backend auth (via `CTXCE_AUTH_ENABLED` and `ctxce auth ...` sessions), uploads and MCP calls are gated by per-user sessions, so multiple IDEs can share the same stack while still having isolated access control and preferences. See **Optional auth with the MCP bridge (PoC)** below for details. - **Flexible transport:** stdio mode works everywhere (even when HTTP ports aren’t reachable), while HTTP mode keeps Claude/Windsurf happy when they want direct URLs; the extension automatically writes the right flavor. - **Centralized logging & health:** when the bridge process runs once per workspace you get a single stream of logs (`Context Engine Upload` output) and a single port to probe for health checks instead of multiple MCP child processes per IDE. - When you run **`Write MCP Config`**, the extension: @@ -64,13 +68,85 @@ MCP bridge (ctx-mcp-bridge) & MCP config lifecycle - `mcpServerMode = bridge`, `mcpTransportMode = http` → **bridge-http**. - `mcpServerMode = direct`, `mcpTransportMode = sse-remote` → **direct-sse** (two stdio `mcp-remote` servers). - `mcpServerMode = direct`, `mcpTransportMode = http` → **direct-http** (two HTTP servers, no bridge). -- In **bridge-stdio**, the configs run `ctxce mcp-serve` via `npx`, passing the workspace path (auto-detected from the uploader target path) plus `--indexer-url` and `--memory-url` derived from the MCP settings. +- In **bridge-stdio**, the configs run the `ctxce mcp-serve` CLI via `npx` (for example, + `npx @context-engine-bridge/context-engine-mcp-bridge ctxce mcp-serve`), passing the + workspace path (auto-detected from the uploader target path) plus `--indexer-url` + and `--memory-url` derived from the MCP settings. - In **bridge-http**, the extension can also **manage the bridge process**: - `autoStartMcpBridge=true` and `mcpServerMode='bridge'` with `mcpTransportMode='http'` → the extension starts `ctxce mcp-http-serve` in the background for the active workspace using `mcpBridgePort`. - The resulting HTTP URL (`http://127.0.0.1:/mcp`) is written into `.mcp.json` and Windsurf’s `mcp_config.json` as the `context-engine` server URL. - In **stdio or direct modes**, the HTTP bridge is **not** auto-started; only the explicit `Start MCP HTTP Bridge` command will launch it. - Bridge settings are **workspace-scoped**, so different workspaces can choose different modes and ports (e.g., one workspace using stdio bridge, another using HTTP bridge on a different port). +Optional auth with the MCP bridge (PoC) +-------------------------------------- + +Auth is **off by default** and fully opt-in. When enabled, the MCP indexer and +memory servers expect a valid `session` id (issued by the backend) on protected +tools. The bridge CLI (`ctxce auth ...`) is the primary way to obtain and cache +that session. + +High-level steps: + +- Enable auth on the remote stack (e.g. dev-remote compose): + - Set `CTXCE_AUTH_ENABLED=1` in the upload/indexer environment. + - Optionally set `CTXCE_AUTH_SHARED_TOKEN` for token-based login. + - Optional: set `CTXCE_AUTH_ADMIN_TOKEN` for creating additional users via `/auth/users`. +- Point the bridge at the auth backend: + - In your local shell (where you run the `ctxce` CLI via `npx`), set `CTXCE_AUTH_BACKEND_URL` + to the upload service URL (e.g. `http://localhost:8004`). + +Token-based login: + +```bash +export CTXCE_AUTH_BACKEND_URL=http://localhost:8004 +export CTXCE_AUTH_TOKEN=change-me-dev-token # must match CTXCE_AUTH_SHARED_TOKEN in the stack + +# Obtain a session and cache it under ~/.ctxce/auth.json +npx @context-engine-bridge/context-engine-mcp-bridge ctxce auth login + +# Check status (optional) +npx @context-engine-bridge/context-engine-mcp-bridge ctxce auth status +``` + +Username/password login (when you have real users): + +- First, create the initial user (once) via `/auth/users` while the auth DB is + empty (no admin token required). This is typically done with a small script + or curl call against the upload service, for example: + + ```bash + curl -X POST http://localhost:8004/auth/users \ + -H "Content-Type: application/json" \ + -d '{"username":"you@example.com","password":"your-password"}' + ``` + +- Then login via the bridge: + +```bash +export CTXCE_AUTH_BACKEND_URL=http://localhost:8004 + +npx @context-engine-bridge/context-engine-mcp-bridge ctxce auth login \ + --username you@example.com \ + --password 'your-password' +``` + +In both modes, the bridge stores the returned `session_id` keyed by +`CTXCE_AUTH_BACKEND_URL` and automatically injects it into all MCP tool calls +as the `session` field. Once you have at least one entry in `~/.ctxce/auth.json`, +the MCP bridge used by the extension will discover and reuse that session +automatically; MCP configs do not need to set `CTXCE_AUTH_BACKEND_URL` in their +`env` blocks. If `CTXCE_AUTH_ENABLED` is off on the backend, these auth settings +are ignored and the bridge behaves exactly as before. + +Session lifetime: + +- By default, issued sessions do **not** expire (`CTXCE_AUTH_SESSION_TTL_SECONDS` + defaults to `0`). +- Operators who want expiry can set `CTXCE_AUTH_SESSION_TTL_SECONDS` (in seconds) + on the backend services. Values `> 0` enable a sliding window: active sessions + are refreshed when validated; values `<= 0` disable expiry. + Workspace-level ctx integration ------------------------------- - The VSIX bundles an `env.example` template plus the ctx hook/CLI so you can dogfood the workflow without copying files manually. diff --git a/vscode-extension/context-engine-uploader/auth_utils.js b/vscode-extension/context-engine-uploader/auth_utils.js new file mode 100644 index 00000000..319825ed --- /dev/null +++ b/vscode-extension/context-engine-uploader/auth_utils.js @@ -0,0 +1,234 @@ +const process = require('process'); + +const _skippedAuthCombos = new Set(); + +function getFetch(deps) { + if (deps && typeof deps.fetchGlobal === 'function') { + return deps.fetchGlobal; + } + try { + if (typeof fetch === 'function') { + return fetch; + } + } catch (_) { + } + return null; +} + +async function ensureAuthIfRequired(endpoint, deps) { + try { + if (!deps || !deps.vscode || !deps.spawnSync || !deps.resolveBridgeCliInvocation || !deps.getWorkspaceFolderPath || !deps.log) { + return; + } + const { vscode, spawnSync, resolveBridgeCliInvocation, getWorkspaceFolderPath, log } = deps; + const fetchFn = getFetch(deps); + const raw = (endpoint || '').trim(); + if (!raw) { + return; + } + if (!fetchFn) { + log('Auth status probe skipped: fetch is not available in this runtime.'); + return; + } + + let baseUrl = raw; + try { + const u = new URL(raw); + baseUrl = `${u.protocol}//${u.host}`; + } catch (_) { + baseUrl = raw.replace(/\/+$/, ''); + } + const statusUrl = `${baseUrl.replace(/\/+$/, '')}/auth/status`; + + let res; + try { + res = await fetchFn(statusUrl, { method: 'GET' }); + } catch (error) { + log(`Auth status probe failed: ${error instanceof Error ? error.message : String(error)}`); + return; + } + if (!res || !res.ok) { + return; + } + + let json; + try { + json = await res.json(); + } catch (_) { + return; + } + if (!json || !json.enabled) { + return; + } + + const invocation = resolveBridgeCliInvocation(); + if (!invocation) { + log('Context Engine Uploader: ctxce CLI not found; skipping auth status check.'); + return; + } + + const backendUrl = baseUrl; + const workspacePath = (typeof getWorkspaceFolderPath === 'function' && getWorkspaceFolderPath()) || ''; + const skipKey = `${backendUrl}::${workspacePath}`; + if (_skippedAuthCombos.has(skipKey)) { + return; + } + + const args = [...invocation.args, 'auth', 'status', '--json', '--backend-url', backendUrl]; + let result; + try { + result = spawnSync(invocation.command, args, { + cwd: getWorkspaceFolderPath() || process.cwd(), + env: { + ...process.env, + CTXCE_AUTH_BACKEND_URL: backendUrl, + }, + encoding: 'utf8', + }); + } catch (error) { + log(`Auth status check failed to run: ${error instanceof Error ? error.message : String(error)}`); + return; + } + + const stdout = (result && result.stdout) || ''; + let parsed; + try { + parsed = stdout ? JSON.parse(stdout) : null; + } catch (_) { + parsed = null; + } + const state = parsed && typeof parsed.state === 'string' ? parsed.state : undefined; + const exitCode = result && typeof result.status === 'number' ? result.status : undefined; + log(`Context Engine Uploader: auth status JSON state=${state || ''} exitCode=${exitCode !== undefined ? exitCode : ''}`); + if (state === 'ok' || result.status === 0) { + return; + } + + const choice = await vscode.window.showInformationMessage( + 'Context Engine: authentication is enabled on the backend but no valid session is available.', + 'Sign In', + 'Skip for now', + ); + if (choice !== 'Sign In') { + _skippedAuthCombos.add(skipKey); + return; + } + + await runAuthLoginFlow(backendUrl, deps); + } catch (error) { + if (deps && typeof deps.log === 'function') { + deps.log(`ensureAuthIfRequired error: ${error instanceof Error ? error.message : String(error)}`); + } + } +} + +async function runAuthLoginFlow(explicitBackendUrl, deps) { + if (!deps || !deps.vscode || !deps.spawn || !deps.resolveBridgeCliInvocation || !deps.getWorkspaceFolderPath || !deps.attachOutput || !deps.log) { + return; + } + const { vscode, spawn, resolveBridgeCliInvocation, getWorkspaceFolderPath, attachOutput, log } = deps; + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + let endpoint = (settings.get('endpoint') || '').trim(); + let backendUrl = explicitBackendUrl || endpoint; + if (!backendUrl) { + vscode.window.showErrorMessage('Context Engine Uploader: backend endpoint is not configured (contextEngineUploader.endpoint).'); + return; + } + + try { + const u = new URL(backendUrl); + backendUrl = `${u.protocol}//${u.host}`; + } catch (_) { + backendUrl = backendUrl.replace(/\/+$/, ''); + } + + const mode = await vscode.window.showQuickPick( + ['Token (shared dev token)', 'Username / password'], + { placeHolder: 'Select Context Engine auth method' }, + ); + if (!mode) { + return; + } + + const invocation = resolveBridgeCliInvocation(); + if (!invocation) { + vscode.window.showErrorMessage('Context Engine Uploader: unable to locate ctxce CLI for auth.'); + return; + } + const cwd = getWorkspaceFolderPath() || process.cwd(); + + if (mode.startsWith('Token')) { + const token = await vscode.window.showInputBox({ + prompt: 'Enter Context Engine shared auth token', + password: true, + ignoreFocusOut: true, + }); + if (!token) { + return; + } + const args = [...invocation.args, 'auth', 'login']; + const env = { + ...process.env, + CTXCE_AUTH_BACKEND_URL: backendUrl, + CTXCE_AUTH_TOKEN: token, + }; + await new Promise(resolve => { + const child = spawn(invocation.command, args, { cwd, env }); + attachOutput(child, 'auth'); + child.on('error', error => { + log(`ctxce auth login (token) failed to start: ${error instanceof Error ? error.message : String(error)}`); + vscode.window.showErrorMessage('Context Engine Uploader: auth login failed to start. See output for details.'); + resolve(); + }); + child.on('close', code => { + if (code === 0) { + vscode.window.showInformationMessage('Context Engine Uploader: auth login successful.'); + } else { + vscode.window.showErrorMessage(`Context Engine Uploader: auth login failed with exit code ${code}. See output for details.`); + } + resolve(); + }); + }); + return; + } + + const username = await vscode.window.showInputBox({ + prompt: 'Enter Context Engine username', + ignoreFocusOut: true, + }); + if (!username) { + return; + } + const password = await vscode.window.showInputBox({ + prompt: 'Enter Context Engine password', + password: true, + ignoreFocusOut: true, + }); + if (!password) { + return; + } + + const args = [...invocation.args, 'auth', 'login', '--backend-url', backendUrl, '--username', username, '--password', password]; + await new Promise(resolve => { + const child = spawn(invocation.command, args, { cwd, env: process.env }); + attachOutput(child, 'auth'); + child.on('error', error => { + log(`ctxce auth login failed to start: ${error instanceof Error ? error.message : String(error)}`); + vscode.window.showErrorMessage('Context Engine Uploader: auth login failed to start. See output for details.'); + resolve(); + }); + child.on('close', code => { + if (code === 0) { + vscode.window.showInformationMessage('Context Engine Uploader: auth login successful.'); + } else { + vscode.window.showErrorMessage(`Context Engine Uploader: auth login failed with exit code ${code}. See output for details.`); + } + resolve(); + }); + }); +} + +module.exports = { + ensureAuthIfRequired, + runAuthLoginFlow, +}; diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index 837d91fe..9edc828d 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -3,6 +3,7 @@ const { spawn, spawnSync } = require('child_process'); const path = require('path'); const fs = require('fs'); const os = require('os'); +const { ensureAuthIfRequired, runAuthLoginFlow } = require('./auth_utils'); let outputChannel; let watchProcess; let forceProcess; @@ -104,6 +105,12 @@ function activate(context) { vscode.window.showErrorMessage('Prompt+ failed. See Context Engine Upload output.'); }); }); + const authLoginDisposable = vscode.commands.registerCommand('contextEngineUploader.authLogin', () => { + runAuthLoginFlow(undefined, buildAuthDeps()).catch(error => { + log(`Auth login failed: ${error instanceof Error ? error.message : String(error)}`); + vscode.window.showErrorMessage('Context Engine Uploader: auth login failed. See output for details.'); + }); + }); const configDisposable = vscode.workspace.onDidChangeConfiguration(event => { if (event.affectsConfiguration('contextEngineUploader') && watchProcess) { runSequence('auto').catch(error => log(`Auto-restart failed: ${error instanceof Error ? error.message : String(error)}`)); @@ -154,6 +161,7 @@ function activate(context) { uploadGitHistoryDisposable, showLogsDisposable, promptEnhanceDisposable, + authLoginDisposable, startBridgeDisposable, stopBridgeDisposable, mcpConfigDisposable, @@ -187,17 +195,36 @@ function activate(context) { } } } +function buildAuthDeps() { + return { + vscode, + spawn, + spawnSync, + resolveBridgeCliInvocation, + getWorkspaceFolderPath, + attachOutput, + log, + fetchGlobal: (typeof fetch === 'function' ? fetch : undefined), + }; +} async function runSequence(mode = 'auto') { const options = resolveOptions(); if (!options) { return; } + + try { + await ensureAuthIfRequired(options.endpoint, buildAuthDeps()); + } catch (error) { + log(`Auth preflight check failed: ${error instanceof Error ? error.message : String(error)}`); + } + const depsSatisfied = await ensurePythonDependencies(options.pythonPath); if (!depsSatisfied) { setStatusBarState('idle'); return; } - // Re-resolve options in case ensurePythonDependencies switched to a private venv interpreter + // Re-resolve options in case ensurePythonDependencies switched to a better interpreter const reoptions = resolveOptions(); if (reoptions) { Object.assign(options, reoptions); diff --git a/vscode-extension/context-engine-uploader/package.json b/vscode-extension/context-engine-uploader/package.json index a670e55f..8c5e7867 100644 --- a/vscode-extension/context-engine-uploader/package.json +++ b/vscode-extension/context-engine-uploader/package.json @@ -15,7 +15,8 @@ "onCommand:contextEngineUploader.start", "onCommand:contextEngineUploader.stop", "onCommand:contextEngineUploader.restart", - "onCommand:contextEngineUploader.promptEnhance" + "onCommand:contextEngineUploader.promptEnhance", + "onCommand:contextEngineUploader.authLogin" ], "main": "./extension.js", "icon": "assets/icon.png", @@ -56,6 +57,10 @@ { "command": "contextEngineUploader.promptEnhance", "title": "Context Engine Uploader: Prompt+ (Unicorn Mode)" + }, + { + "command": "contextEngineUploader.authLogin", + "title": "Context Engine Uploader: Sign In (ctxce auth)" } ], "configuration": { From d9101e8a44d3efb213ebd880d8de8db632c6234e Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 00:27:17 +0000 Subject: [PATCH 20/55] fix: Propagate auth config to docker-compose --- docker-compose.dev-remote.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docker-compose.dev-remote.yml b/docker-compose.dev-remote.yml index 437ec4a0..8efac541 100644 --- a/docker-compose.dev-remote.yml +++ b/docker-compose.dev-remote.yml @@ -33,6 +33,12 @@ services: - FASTMCP_HOST=${FASTMCP_HOST} - FASTMCP_PORT=${FASTMCP_PORT} - QDRANT_URL=${QDRANT_URL} + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} - COLLECTION_NAME=${COLLECTION_NAME} - PATH_EMIT_MODE=auto - HF_HOME=/work/.cache/huggingface @@ -75,6 +81,12 @@ services: - FASTMCP_HOST=${FASTMCP_HOST} - FASTMCP_INDEXER_PORT=${FASTMCP_INDEXER_PORT} - QDRANT_URL=${QDRANT_URL} + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} - REFRAG_DECODER=${REFRAG_DECODER:-1} - REFRAG_RUNTIME=${REFRAG_RUNTIME:-llamacpp} - GLM_API_KEY=${GLM_API_KEY} @@ -120,6 +132,12 @@ services: - FASTMCP_PORT=8000 - FASTMCP_TRANSPORT=${FASTMCP_HTTP_TRANSPORT} - QDRANT_URL=${QDRANT_URL} + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} - COLLECTION_NAME=${COLLECTION_NAME} - PATH_EMIT_MODE=auto - HF_HOME=/work/.cache/huggingface @@ -162,6 +180,12 @@ services: - FASTMCP_INDEXER_PORT=8001 - FASTMCP_TRANSPORT=${FASTMCP_HTTP_TRANSPORT} - QDRANT_URL=${QDRANT_URL} + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} - REFRAG_DECODER=${REFRAG_DECODER:-1} - REFRAG_RUNTIME=${REFRAG_RUNTIME:-llamacpp} - GLM_API_KEY=${GLM_API_KEY} @@ -334,6 +358,12 @@ services: - WORKDIR=/work - MAX_BUNDLE_SIZE_MB=100 - UPLOAD_TIMEOUT_SECS=300 + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} # Indexing configuration - COLLECTION_NAME=${COLLECTION_NAME} From 49245c03c4543a2d65cfecdeb4cde1348cff77a3 Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 00:27:17 +0000 Subject: [PATCH 21/55] fix: Update remote upload clients to use auth sessions --- scripts/remote_upload_client.py | 115 +++++++++++++-------------- scripts/standalone_upload_client.py | 117 +++++++++++++--------------- 2 files changed, 109 insertions(+), 123 deletions(-) diff --git a/scripts/remote_upload_client.py b/scripts/remote_upload_client.py index 37646e58..e65b69ed 100644 --- a/scripts/remote_upload_client.py +++ b/scripts/remote_upload_client.py @@ -493,6 +493,7 @@ def _get_temp_bundle_dir(self) -> Path: if not self.temp_dir: self.temp_dir = tempfile.mkdtemp(prefix="delta_bundle_") return Path(self.temp_dir) + # CLI is stateless - sequence tracking is handled by server def detect_file_changes(self, changed_paths: List[Path]) -> Dict[str, List]: @@ -820,7 +821,7 @@ def create_delta_bundle(self, changes: Dict[str, List]) -> Tuple[str, Dict[str, "workspace_path": self.workspace_path, "collection_name": self.collection_name, "created_at": created_at, - # CLI is stateless - server will assign sequence numbers + # CLI is stateless - server handles sequence numbers "sequence_number": None, # Server will assign "parent_sequence": None, # Server will determine "operations": { @@ -899,76 +900,68 @@ def upload_bundle(self, bundle_path: str, manifest: Dict[str, Any]) -> Dict[str, # Check bundle size (server-side enforcement) bundle_size = os.path.getsize(bundle_path) - with open(bundle_path, 'rb') as bundle_file: - files = { - 'bundle': (f"{manifest['bundle_id']}.tar.gz", bundle_file, 'application/gzip') - } + files = { + "bundle": open(bundle_path, "rb"), + } + data = { + "workspace_path": self._translate_to_container_path(self.workspace_path), + "collection_name": self.collection_name, + "sequence_number": manifest.get("sequence_number"), + "force": False, + "source_path": self.workspace_path, + "logical_repo_id": _compute_logical_repo_id(self.workspace_path), + } - data = { - 'workspace_path': self._translate_to_container_path(self.workspace_path), - 'collection_name': self.collection_name, - # CLI is stateless - server handles sequence numbers - 'force': 'false', - 'source_path': self.workspace_path, - } - if getattr(self, "logical_repo_id", None): - data['logical_repo_id'] = self.logical_repo_id + sess = get_auth_session(self.upload_endpoint) + if sess: + data["session"] = sess - logger.info(f"[remote_upload] Uploading bundle {manifest['bundle_id']} (size: {bundle_size} bytes)") + if getattr(self, "logical_repo_id", None): + data['logical_repo_id'] = self.logical_repo_id - response = self.session.post( - f"{self.upload_endpoint}/api/v1/delta/upload", - files=files, - data=data, - timeout=(10, self.timeout) - ) + logger.info(f"[remote_upload] Uploading bundle {manifest['bundle_id']} (size: {bundle_size} bytes)") - if response.status_code == 200: - result = response.json() - logger.info(f"[remote_upload] Successfully uploaded bundle {manifest['bundle_id']}") + response = self.session.post( + f"{self.upload_endpoint}/api/v1/delta/upload", + files=files, + data=data, + timeout=(10, self.timeout) + ) - seq = None + result = None + try: + result = response.json() + except Exception: + result = None + + if response.status_code == 200 and isinstance(result, dict) and result.get("success", False): + logger.info(f"[remote_upload] Successfully uploaded bundle {manifest['bundle_id']}") + seq = result.get("sequence_number") + if seq is not None: try: - seq = result.get("sequence_number") + manifest["sequence"] = seq except Exception: - seq = None - if seq is not None: - try: - manifest["sequence"] = seq - except Exception: - pass - - poll_result = self._poll_after_timeout(manifest) - if poll_result.get("success"): - combined = dict(result) - for k, v in poll_result.items(): - if k in ("success", "error"): - continue - if k not in combined: - combined[k] = v - return combined - - logger.warning("[remote_upload] Upload accepted but polling did not confirm processing; returning original result") - return result - - # Handle error - error_msg = f"Upload failed with status {response.status_code}" - try: - error_detail = response.json() - error_detail_msg = error_detail.get('error', {}).get('message', 'Unknown error') - error_msg += f": {error_detail_msg}" - error_code = error_detail.get('error', {}).get('code', 'HTTP_ERROR') - except: - error_msg += f": {response.text[:200]}" - error_code = "HTTP_ERROR" + pass + return result + + # Handle error + error_msg = f"Upload failed with status {response.status_code}" + try: + error_detail = result if isinstance(result, dict) else response.json() + error_detail_msg = error_detail.get('error', {}).get('message', 'Unknown error') + error_msg += f": {error_detail_msg}" + error_code = error_detail.get('error', {}).get('code', 'HTTP_ERROR') + except Exception: + error_msg += f": {response.text[:200]}" + error_code = "HTTP_ERROR" - last_error = {"success": False, "error": {"code": error_code, "message": error_msg, "status_code": response.status_code}} + last_error = {"success": False, "error": {"code": error_code, "message": error_msg, "status_code": response.status_code}} - # Don't retry on client errors (except 429) - if 400 <= response.status_code < 500 and response.status_code != 429: - return last_error + # Don't retry on client errors (except 429) + if 400 <= response.status_code < 500 and response.status_code != 429: + return last_error - logger.warning(f"[remote_upload] Upload attempt {attempt + 1} failed: {error_msg}") + logger.warning(f"[remote_upload] Upload attempt {attempt + 1} failed: {error_msg}") except requests.exceptions.ConnectTimeout as e: last_error = {"success": False, "error": {"code": "TIMEOUT_ERROR", "message": f"Upload timeout: {str(e)}"}} diff --git a/scripts/standalone_upload_client.py b/scripts/standalone_upload_client.py index 958d2ae1..2c90a806 100644 --- a/scripts/standalone_upload_client.py +++ b/scripts/standalone_upload_client.py @@ -27,6 +27,7 @@ import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry +from scripts.upload_auth_utils import get_auth_session # Configure logging logging.basicConfig(level=logging.INFO) @@ -656,6 +657,7 @@ def _get_temp_bundle_dir(self) -> Path: if not self.temp_dir: self.temp_dir = tempfile.mkdtemp(prefix="delta_bundle_") return Path(self.temp_dir) + # CLI is stateless - sequence tracking is handled by server def detect_file_changes(self, changed_paths: List[Path]) -> Dict[str, List]: @@ -981,7 +983,7 @@ def create_delta_bundle(self, changes: Dict[str, List]) -> Tuple[str, Dict[str, "workspace_path": self.workspace_path, "collection_name": self.collection_name, "created_at": created_at, - # CLI is stateless - server will assign sequence numbers + # CLI is stateless - server handles sequence numbers "sequence_number": None, # Server will assign "parent_sequence": None, # Server will determine "operations": { @@ -1031,8 +1033,7 @@ def create_delta_bundle(self, changes: Dict[str, List]) -> Tuple[str, Dict[str, return str(bundle_path), manifest def upload_bundle(self, bundle_path: str, manifest: Dict[str, Any]) -> Dict[str, Any]: - """ - Upload delta bundle to remote server with exponential backoff retry. + """Upload delta bundle to remote server with exponential backoff retry. Args: bundle_path: Path to the bundle tarball @@ -1058,76 +1059,68 @@ def upload_bundle(self, bundle_path: str, manifest: Dict[str, Any]) -> Dict[str, # Check bundle size (server-side enforcement) bundle_size = os.path.getsize(bundle_path) - with open(bundle_path, 'rb') as bundle_file: - files = { - 'bundle': (f"{manifest['bundle_id']}.tar.gz", bundle_file, 'application/gzip') - } + files = { + "bundle": open(bundle_path, "rb"), + } + data = { + "workspace_path": self._translate_to_container_path(self.workspace_path), + "collection_name": self.collection_name, + "sequence_number": manifest.get("sequence_number"), + "force": False, + "source_path": self.workspace_path, + "logical_repo_id": _compute_logical_repo_id(self.workspace_path), + } - data = { - 'workspace_path': self._translate_to_container_path(self.workspace_path), - 'collection_name': self.collection_name, - # CLI is stateless - server handles sequence numbers - 'force': 'false', - 'source_path': self.workspace_path, - } + sess = get_auth_session(self.upload_endpoint) + if sess: + data["session"] = sess - if getattr(self, "logical_repo_id", None): - data['logical_repo_id'] = self.logical_repo_id + if getattr(self, "logical_repo_id", None): + data['logical_repo_id'] = self.logical_repo_id - logger.info(f"[remote_upload] Uploading bundle {manifest['bundle_id']} (size: {bundle_size} bytes)") + logger.info(f"[remote_upload] Uploading bundle {manifest['bundle_id']} (size: {bundle_size} bytes)") - response = self.session.post( - f"{self.upload_endpoint}/api/v1/delta/upload", - files=files, - data=data, - timeout=(10, self.timeout) - ) + response = self.session.post( + f"{self.upload_endpoint}/api/v1/delta/upload", + files=files, + data=data, + timeout=(10, self.timeout) + ) - if response.status_code == 200: - result = response.json() - logger.info(f"[remote_upload] Successfully uploaded bundle {manifest['bundle_id']}") + result = None + try: + result = response.json() + except Exception: + result = None - seq = None + if response.status_code == 200 and isinstance(result, dict) and result.get("success", False): + logger.info(f"[remote_upload] Successfully uploaded bundle {manifest['bundle_id']}") + seq = result.get("sequence_number") + if seq is not None: try: - seq = result.get("sequence_number") + manifest["sequence"] = seq except Exception: - seq = None - if seq is not None: - try: - manifest["sequence"] = seq - except Exception: - pass - - poll_result = self._poll_after_timeout(manifest) - if poll_result.get("success"): - combined = dict(result) - for k, v in poll_result.items(): - if k in ("success", "error"): - continue - if k not in combined: - combined[k] = v - return combined - - logger.warning("[remote_upload] Upload accepted but polling did not confirm processing; returning original result") - return result - # Handle error - error_msg = f"Upload failed with status {response.status_code}" - try: - error_detail = response.json() - error_detail_msg = error_detail.get('error', {}).get('message', 'Unknown error') - error_msg += f": {error_detail_msg}" - error_code = error_detail.get('error', {}).get('code', 'HTTP_ERROR') - except: - error_msg += f": {response.text[:200]}" - error_code = "HTTP_ERROR" + pass + return result + + # Handle error + error_msg = f"Upload failed with status {response.status_code}" + try: + error_detail = result if isinstance(result, dict) else response.json() + error_detail_msg = error_detail.get('error', {}).get('message', 'Unknown error') + error_msg += f": {error_detail_msg}" + error_code = error_detail.get('error', {}).get('code', 'HTTP_ERROR') + except Exception: + error_msg += f": {response.text[:200]}" + error_code = "HTTP_ERROR" - last_error = {"success": False, "error": {"code": error_code, "message": error_msg, "status_code": response.status_code}} + last_error = {"success": False, "error": {"code": error_code, "message": error_msg, "status_code": response.status_code}} - # Don't retry on client errors (except 429) - if 400 <= response.status_code < 500 and response.status_code != 429: - return last_error + # Don't retry on client errors (except 429) + if 400 <= response.status_code < 500 and response.status_code != 429: + return last_error - logger.warning(f"[remote_upload] Upload attempt {attempt + 1} failed: {error_msg}") + logger.warning(f"[remote_upload] Upload attempt {attempt + 1} failed: {error_msg}") except requests.exceptions.ConnectTimeout as e: last_error = {"success": False, "error": {"code": "TIMEOUT_ERROR", "message": f"Upload timeout: {str(e)}"}} From 0a3f33789626efb1550bfbc9e01d9fcbc39f5500 Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 00:27:17 +0000 Subject: [PATCH 22/55] fix: Enable smart symbol reindexing by default --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 9ed72435..2780cf5f 100644 --- a/.env.example +++ b/.env.example @@ -189,7 +189,7 @@ MEMORY_AUTODETECT=1 MEMORY_COLLECTION_TTL_SECS=300 # Smarter re-indexing for symbol cache, reuse embeddings and reduce decoder/pseudo tags to re-index -SMART_SYMBOL_REINDEXING=0 +SMART_SYMBOL_REINDEXING=1 # Watcher-safe defaults (recommended) # Applied to watcher via compose; uncomment to apply globally. From a03e6365d42d84c90e83346ceda1a033679f02de Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 00:27:17 +0000 Subject: [PATCH 23/55] feat: Remove scaffolding from writeMcpConfig and add default llama cpp model to ctx config --- vscode-extension/context-engine-uploader/extension.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index 9edc828d..a74503cb 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -1299,11 +1299,6 @@ async function writeMcpConfig() { } } } - if (!wroteAny && !hookWrote) { - log('Context Engine Uploader: MCP config write skipped (no targets succeeded).'); - } - - // Optionally scaffold ctx_config.json and .env using the inferred collection if (settings.get('scaffoldCtxConfig', true)) { try { await writeCtxConfig(); @@ -1500,6 +1495,12 @@ async function scaffoldCtxConfigFiles(workspaceDir, collectionName) { ctxChanged = true; } } + if (decoderRuntime === 'llamacpp') { + if (ctxConfig.llamacpp_model === undefined) { + ctxConfig.llamacpp_model = 'llamacpp-4.6'; + ctxChanged = true; + } + } if (ctxChanged) { fs.writeFileSync(ctxConfigPath, JSON.stringify(ctxConfig, null, 2) + '\n', 'utf8'); if (notifiedDefault) { From ef0e10366746765261d9260c3e6a3164bb92a3fa Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 00:27:17 +0000 Subject: [PATCH 24/55] fix: bump extension version --- vscode-extension/context-engine-uploader/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vscode-extension/context-engine-uploader/package.json b/vscode-extension/context-engine-uploader/package.json index 8c5e7867..00f83196 100644 --- a/vscode-extension/context-engine-uploader/package.json +++ b/vscode-extension/context-engine-uploader/package.json @@ -2,7 +2,7 @@ "name": "context-engine-uploader", "displayName": "Context Engine Uploader", "description": "Runs the Context-Engine remote upload client with a force sync on startup followed by watch mode. Requires Python with pip install requests urllib3 charset_normalizer.", - "version": "0.1.31", + "version": "0.1.32", "publisher": "context-engine", "engines": { "vscode": "^1.85.0" From ca527237d395edfc30f656043a22c2a254393f31 Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 11:06:43 +0000 Subject: [PATCH 25/55] fix: stop mangling MCP URLs in bridge mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove /sse → /mcp rewrite from normalizeBridgeUrl Update MCP URL defaults and bridge fallbacks to 8003/8002 /mcp HTTP endpoints Ensure bridge modes consistently talk HTTP to the backend MCP pair; SSE remains for direct mcp-remote only --- .../context-engine-uploader/extension.js | 17 ++++------------- .../context-engine-uploader/package.json | 6 +++--- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index a74503cb..d75e08f8 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -1173,15 +1173,6 @@ function normalizeBridgeUrl(url) { if (!trimmed) { return ''; } - try { - const parsed = new URL(trimmed); - if (parsed.pathname.endsWith('/sse')) { - parsed.pathname = parsed.pathname.replace(/\/sse$/, '/mcp'); - return parsed.toString(); - } - } catch (_) { - // fall through to return trimmed - } return trimmed; } @@ -1258,8 +1249,8 @@ async function writeMcpConfig() { } } - let indexerUrl = (settings.get('mcpIndexerUrl') || 'http://localhost:8001/sse').trim(); - let memoryUrl = (settings.get('mcpMemoryUrl') || 'http://localhost:8000/sse').trim(); + let indexerUrl = (settings.get('mcpIndexerUrl') || 'http://localhost:8003/mcp').trim(); + let memoryUrl = (settings.get('mcpMemoryUrl') || 'http://localhost:8002/mcp').trim(); if (serverMode === 'bridge') { indexerUrl = normalizeBridgeUrl(indexerUrl); memoryUrl = normalizeBridgeUrl(memoryUrl); @@ -1745,8 +1736,8 @@ function resolveHttpBridgeOptions() { vscode.window.showErrorMessage('Context Engine Uploader: open a workspace or set contextEngineUploader.targetPath before starting HTTP MCP bridge.'); return undefined; } - let indexerUrl = (settings.get('mcpIndexerUrl') || 'http://localhost:8001/sse').trim(); - let memoryUrl = (settings.get('mcpMemoryUrl') || '').trim(); + let indexerUrl = (settings.get('mcpIndexerUrl') || 'http://localhost:8003/mcp').trim(); + let memoryUrl = (settings.get('mcpMemoryUrl') || 'http://localhost:8002/mcp').trim(); indexerUrl = normalizeBridgeUrl(indexerUrl); memoryUrl = normalizeBridgeUrl(memoryUrl); let port = Number(settings.get('mcpBridgePort') || 30810); diff --git a/vscode-extension/context-engine-uploader/package.json b/vscode-extension/context-engine-uploader/package.json index 00f83196..98c5c8b6 100644 --- a/vscode-extension/context-engine-uploader/package.json +++ b/vscode-extension/context-engine-uploader/package.json @@ -192,7 +192,7 @@ }, "contextEngineUploader.mcpBridgeLocalOnly": { "type": "boolean", - "default": true, + "default": false, "description": "Development toggle. When true (default) the extension prefers local bridge binaries resolved from mcpBridgeBinPath or CTXCE_BRIDGE_BIN before falling back to the published npm build via npx." }, "contextEngineUploader.mcpServerMode": { @@ -207,12 +207,12 @@ }, "contextEngineUploader.mcpIndexerUrl": { "type": "string", - "default": "http://localhost:8001/sse", + "default": "http://localhost:8003/mcp", "description": "Claude Code MCP server URL for the Qdrant indexer. Used when writing MCP configs." }, "contextEngineUploader.mcpMemoryUrl": { "type": "string", - "default": "http://localhost:8000/sse", + "default": "http://localhost:8002/mcp", "description": "Claude Code MCP server URL for the memory/search MCP server. Used when writing MCP configs." }, "contextEngineUploader.ctxIndexerUrl": { From b37a56122aa96be4140971dd566734c006a6ec6f Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 11:25:50 +0000 Subject: [PATCH 26/55] Fix: Windsurf HTTP bridge hot reload mcp config When the extension auto-starts the HTTP bridge in bridge-http mode, drop context-engine from Windsurf mcp_config.json and re-write it after the bridge comes back --- .../context-engine-uploader/extension.js | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index d75e08f8..98e746f9 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -481,6 +481,25 @@ function scheduleMcpConfigRefreshAfterBridge(delayMs = 1500) { clearTimeout(pendingBridgeConfigTimer); pendingBridgeConfigTimer = undefined; } + // For bridge-http mode started by the extension, Windsurf needs the + // "context-engine" MCP server entry removed and then re-added once the + // HTTP bridge is ready. Best-effort removal happens immediately here; + // writeMcpConfig() below will re-write configs after the bridge comes up. + try { + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + const windsurfEnabled = settings.get('mcpWindsurfEnabled', false); + const transportModeRaw = settings.get('mcpTransportMode') || 'sse-remote'; + const serverModeRaw = settings.get('mcpServerMode') || 'bridge'; + const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; + const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + if (windsurfEnabled && serverMode === 'bridge' && transportMode === 'http') { + removeContextEngineFromWindsurfConfig().catch(error => { + log(`Context Engine Uploader: failed to remove context-engine from Windsurf MCP config before HTTP bridge restart: ${error instanceof Error ? error.message : String(error)}`); + }); + } + } catch (error) { + log(`Context Engine Uploader: failed to prepare Windsurf MCP removal before HTTP bridge restart: ${error instanceof Error ? error.message : String(error)}`); + } pendingBridgeConfigTimer = setTimeout(() => { pendingBridgeConfigTimer = undefined; log('Context Engine Uploader: HTTP bridge ready; refreshing MCP configs.'); @@ -492,6 +511,48 @@ function scheduleMcpConfigRefreshAfterBridge(delayMs = 1500) { log(`Context Engine Uploader: failed to schedule MCP config refresh: ${error instanceof Error ? error.message : String(error)}`); } } + +async function removeContextEngineFromWindsurfConfig() { + try { + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + const customPath = (settings.get('windsurfMcpPath') || '').trim(); + const configPath = customPath || getDefaultWindsurfMcpPath(); + if (!configPath) { + return; + } + if (!fs.existsSync(configPath)) { + // Nothing to remove yet. + return; + } + let config = { mcpServers: {} }; + try { + const raw = fs.readFileSync(configPath, 'utf8'); + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === 'object') { + config = parsed; + } + } catch (error) { + log(`Context Engine Uploader: failed to parse Windsurf mcp_config.json when removing context-engine: ${error instanceof Error ? error.message : String(error)}`); + return; + } + if (!config.mcpServers || typeof config.mcpServers !== 'object') { + return; + } + if (!config.mcpServers['context-engine']) { + return; + } + delete config.mcpServers['context-engine']; + try { + const json = JSON.stringify(config, null, 2) + '\n'; + fs.writeFileSync(configPath, json, 'utf8'); + log(`Context Engine Uploader: removed context-engine server from Windsurf MCP config at ${configPath} before HTTP bridge restart.`); + } catch (error) { + log(`Context Engine Uploader: failed to write Windsurf mcp_config.json when removing context-engine: ${error instanceof Error ? error.message : String(error)}`); + } + } catch (error) { + log(`Context Engine Uploader: error while removing context-engine from Windsurf MCP config: ${error instanceof Error ? error.message : String(error)}`); + } +} function ensureTargetPathConfigured() { const config = vscode.workspace.getConfiguration('contextEngineUploader'); const current = (config.get('targetPath') || '').trim(); From 9f231e8c8d5e7ae5f66d0afe58cbc650c2620d4c Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 11:45:25 +0000 Subject: [PATCH 27/55] fix(upload-service): prevent recursive dev-workspace nesting and env-specific repo detection - Add defensive guard in process_delta_bundle to detect rel_path values that already include the workspace slug (-) and treat them as a hard error, preventing creation of nested slug/slug/... directories - Keep a clear error log message when this guard triggers so misconfigured clients (e.g. using dev-workspace mirrors as workspace roots) are easy to diagnose - Remove hard-coded /home/coder/project/Context-Engine/dev-workspace from workspace_state._detect_repo_name_from_path candidate roots so repo detection is driven by WATCH_ROOT, WORKSPACE_PATH, /work, and HOST_ROOT, instead of a repo-specific path baked into the code --- scripts/upload_service.py | 16 ++++++++++++++++ scripts/workspace_state.py | 1 - 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/upload_service.py b/scripts/upload_service.py index 8de5bfed..a5f70e7f 100644 --- a/scripts/upload_service.py +++ b/scripts/upload_service.py @@ -267,6 +267,7 @@ def process_delta_bundle(workspace_path: str, bundle_path: Path, manifest: Dict[ workspace_key = get_workspace_key(workspace_path) workspace = Path(WORK_DIR) / f"{repo_name}-{workspace_key}" workspace.mkdir(parents=True, exist_ok=True) + slug_repo_name = f"{repo_name}-{workspace_key}" with tarfile.open(bundle_path, "r:gz") as tar: # Extract operations metadata @@ -317,6 +318,21 @@ def process_delta_bundle(workspace_path: str, bundle_path: Path, manifest: Dict[ operations_count["skipped"] += 1 continue + # Defensive guard: if rel_path already starts with the slugged + # repo name (e.g. "-/"), writing to workspace / rel_path + # would create a nested slug directory ("slug/slug/..."). This is + # almost certainly a misconfigured client using a dev-workspace + # mirror as the workspace root. Treat this as a hard error so the + # bundle does not silently create recursive structures. + # TODO: http error code/msg for extension toast? + if rel_path == slug_repo_name or rel_path.startswith(slug_repo_name + "/"): + msg = ( + f"[upload_service] Refusing to apply operation {op_type} for suspicious path {rel_path} " + f"which already contains workspace slug {slug_repo_name}" + ) + logger.error(msg) + raise ValueError(msg) + target_path = workspace / rel_path try: diff --git a/scripts/workspace_state.py b/scripts/workspace_state.py index da7d6b73..d31bafb3 100644 --- a/scripts/workspace_state.py +++ b/scripts/workspace_state.py @@ -626,7 +626,6 @@ def _detect_repo_name_from_path(path: Path) -> str: os.environ.get("WORKSPACE_PATH"), "/work", os.environ.get("HOST_ROOT"), - "/home/coder/project/Context-Engine/dev-workspace", ): if not root_str: continue From 07220b628f35b286346983ace215c54f182ec996 Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 12:28:51 +0000 Subject: [PATCH 28/55] fix(extension): bundle auth helper and decouple standalone upload client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update standalone_upload_client.py to import a local upload_auth_utils module when bundled with the extension, avoiding any hard dependency on a 'scripts' package in the extension context - Bundle upload_auth_utils.py into the packaged extension alongside standalone_upload_client.py via build/build.sh so the bundled client has a self-contained auth helper - Gracefully degrade to “no session” when auth helper is unavailable instead of crashing with ModuleNotFoundError --- scripts/standalone_upload_client.py | 7 ++++++- vscode-extension/build/build.sh | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/standalone_upload_client.py b/scripts/standalone_upload_client.py index 2c90a806..47b6d810 100644 --- a/scripts/standalone_upload_client.py +++ b/scripts/standalone_upload_client.py @@ -27,7 +27,12 @@ import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry -from scripts.upload_auth_utils import get_auth_session + +try: + from upload_auth_utils import get_auth_session # type: ignore[import] +except ImportError: + def get_auth_session(upload_endpoint: str) -> str: + return "" # Configure logging logging.basicConfig(level=logging.INFO) diff --git a/vscode-extension/build/build.sh b/vscode-extension/build/build.sh index 9bb2519b..60e0028c 100755 --- a/vscode-extension/build/build.sh +++ b/vscode-extension/build/build.sh @@ -14,6 +14,7 @@ CTX_SRC="$SCRIPT_DIR/../../scripts/ctx.py" ROUTER_SRC="$SCRIPT_DIR/../../scripts/mcp_router.py" REFRAG_SRC="$SCRIPT_DIR/../../scripts/refrag_glm.py" ENV_EXAMPLE_SRC="$SCRIPT_DIR/../../.env.example" +AUTH_SRC="$SCRIPT_DIR/../../scripts/upload_auth_utils.py" cleanup() { rm -rf "$STAGE_DIR" @@ -54,6 +55,11 @@ if [[ -f "$REFRAG_SRC" ]]; then cp "$REFRAG_SRC" "$STAGE_DIR/refrag_glm.py" fi +# Bundle auth helper used by standalone_upload_client.py +if [[ -f "$AUTH_SRC" ]]; then + cp "$AUTH_SRC" "$STAGE_DIR/upload_auth_utils.py" +fi + if [[ -f "$ENV_EXAMPLE_SRC" ]]; then cp "$ENV_EXAMPLE_SRC" "$STAGE_DIR/env.example" fi From eca498f8ec8da046f26c18352c49c675d15c5529 Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 12:30:13 +0000 Subject: [PATCH 29/55] chore(extension): declare MCP bridge commands in package.json - Add command contributions for Start/Stop MCP HTTP Bridge so they appear in the Command Palette as documented --- vscode-extension/context-engine-uploader/package.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vscode-extension/context-engine-uploader/package.json b/vscode-extension/context-engine-uploader/package.json index 98c5c8b6..9cff4cc2 100644 --- a/vscode-extension/context-engine-uploader/package.json +++ b/vscode-extension/context-engine-uploader/package.json @@ -42,6 +42,14 @@ "command": "contextEngineUploader.uploadGitHistory", "title": "Context Engine Uploader: Upload Git History (force sync bundle)" }, + { + "command": "contextEngineUploader.startMcpHttpBridge", + "title": "Context Engine Uploader: Start MCP HTTP Bridge" + }, + { + "command": "contextEngineUploader.stopMcpHttpBridge", + "title": "Context Engine Uploader: Stop MCP HTTP Bridge" + }, { "command": "contextEngineUploader.writeMcpConfig", "title": "Context Engine Uploader: Write MCP Config (.mcp.json)" From fc3fb0f2811478dfd61fd44ce395da02075d85a6 Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 12:39:37 +0000 Subject: [PATCH 30/55] fix(extension): honor configured MCP indexer URL in Prompt+ ctx invocation - When launching ctx.py for Prompt+, pass CP_INDEXER_URL from ctxIndexerUrl/mcpIndexerUrl settings so it no longer defaults to localhost - Preserve decoder URL override via decoderUrl as before --- vscode-extension/context-engine-uploader/extension.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index 98e746f9..eea96af5 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -977,6 +977,15 @@ async function enhanceSelectionWithUnicorn() { if (decoderUrl) { env.DECODER_URL = decoderUrl; } + try { + const cfg = vscode.workspace.getConfiguration('contextEngineUploader'); + const idxUrlRaw = (cfg.get('ctxIndexerUrl') || cfg.get('mcpIndexerUrl') || '').trim(); + if (idxUrlRaw) { + env.MCP_INDEXER_URL = idxUrlRaw; + } + } catch (_) { + // ignore config read failures; fall back to defaults + } try { const cfg = vscode.workspace.getConfiguration('contextEngineUploader'); const useGpuDecoder = cfg.get('useGpuDecoder', false); From d1b85be566a3bf3a0d1e787e730aca2489eae22a Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 17:50:10 +0000 Subject: [PATCH 31/55] feat(auth): Allow unauthenticated login for development --- .env.example | 3 + ctx-mcp-bridge/src/authCli.js | 98 +++++++++++++++++-- scripts/auth_backend.py | 15 +++ .../context-engine-uploader/README.md | 15 ++- .../context-engine-uploader/extension.js | 28 +++++- 5 files changed, 140 insertions(+), 19 deletions(-) diff --git a/.env.example b/.env.example index 2780cf5f..15f927eb 100644 --- a/.env.example +++ b/.env.example @@ -253,6 +253,9 @@ INFO_REQUEST_CONTEXT_LINES=5 # sessions when the provided token matches this value. # CTXCE_AUTH_SHARED_TOKEN=change-me-dev-token +# Create "open-dev" tokens when a shared token is unset, basically allows requesting a session with no real auth +# CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=0 + # Optional admin token for creating additional users via /auth/users once the # first user has been bootstrapped. If unset, only the initial user can be # created (no additional users). diff --git a/ctx-mcp-bridge/src/authCli.js b/ctx-mcp-bridge/src/authCli.js index eadb0bee..45bc86b2 100644 --- a/ctx-mcp-bridge/src/authCli.js +++ b/ctx-mcp-bridge/src/authCli.js @@ -1,5 +1,5 @@ import process from "node:process"; -import { loadAuthEntry, saveAuthEntry, deleteAuthEntry } from "./authConfig.js"; +import { loadAuthEntry, saveAuthEntry, deleteAuthEntry, loadAnyAuthEntry } from "./authConfig.js"; function parseAuthArgs(args) { let backendUrl = process.env.CTXCE_AUTH_BACKEND_URL || ""; @@ -19,9 +19,16 @@ function parseAuthArgs(args) { i += 1; continue; } - if ((a === "--username" || a === "--user") && i + 1 < args.length) { - username = args[i + 1]; - i += 1; + if (a === "--username" || a === "--user") { + const hasNext = i + 1 < args.length; + const next = hasNext ? String(args[i + 1]) : ""; + if (hasNext && !next.startsWith("-")) { + username = args[i + 1]; + i += 1; + } else { + console.error("[ctxce] Missing value for --username/--user; expected a username."); + process.exit(1); + } continue; } if ((a === "--password" || a === "--pass") && i + 1 < args.length) { @@ -41,6 +48,11 @@ function getBackendUrl(backendUrl) { return (backendUrl || process.env.CTXCE_AUTH_BACKEND_URL || "").trim(); } +function getDefaultUploadBackend() { + // Default to upload service when nothing else is configured + return (process.env.CTXCE_UPLOAD_ENDPOINT || process.env.UPLOAD_ENDPOINT || "http://localhost:8004").trim(); +} + function requireBackendUrl(backendUrl) { const url = getBackendUrl(backendUrl); if (!url) { @@ -67,7 +79,26 @@ function outputJsonStatus(url, state, entry, rawExpires) { async function doLogin(args) { const { backendUrl, token, username, password } = parseAuthArgs(args); - const url = requireBackendUrl(backendUrl); + let url = getBackendUrl(backendUrl); + if (!url) { + // Fallback: use any stored auth entry when no backend is provided + const any = loadAnyAuthEntry(); + if (any && any.backendUrl) { + url = any.backendUrl; + console.error("[ctxce] Using stored backend for login:", url); + } + } + if (!url) { + // Final fallback: default upload endpoint (extension's upload endpoint or localhost:8004) + url = getDefaultUploadBackend(); + if (url) { + console.error("[ctxce] Using default upload backend for login:", url); + } + } + if (!url) { + console.error("[ctxce] Auth backend URL not configured. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + process.exit(1); + } const trimmedUser = (username || "").trim(); const usePassword = trimmedUser && (password || "").length > 0; @@ -124,13 +155,30 @@ async function doLogin(args) { async function doStatus(args) { const { backendUrl, outputJson } = parseAuthArgs(args); - const url = getBackendUrl(backendUrl); + let url = getBackendUrl(backendUrl); + let usedFallback = false; + if (!url) { + // Fallback: use any stored auth entry when no backend is provided + const any = loadAnyAuthEntry(); + if (any && any.backendUrl) { + url = any.backendUrl; + usedFallback = true; + } + } + if (!url) { + // Final fallback: default upload endpoint + url = getDefaultUploadBackend(); + if (url) { + usedFallback = true; + console.error("[ctxce] Using default upload backend for status:", url); + } + } if (!url) { if (outputJson) { outputJsonStatus("", "missing_backend", null, null); process.exit(1); } - console.error("[ctxce] Auth backend URL not configured. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + console.error("[ctxce] Auth backend URL not configured and no stored sessions found. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); process.exit(1); } let entry; @@ -149,7 +197,11 @@ async function doStatus(args) { outputJsonStatus(url, "missing", null, rawExpires); process.exit(1); } - console.error("[ctxce] Not logged in for", url); + if (usedFallback) { + console.error("[ctxce] Not logged in for stored backend", url); + } else { + console.error("[ctxce] Not logged in for", url); + } process.exit(1); } @@ -158,7 +210,11 @@ async function doStatus(args) { outputJsonStatus(url, "expired", entry, rawExpires); process.exit(2); } - console.error("[ctxce] Stored auth session appears expired for", url); + if (usedFallback) { + console.error("[ctxce] Stored auth session appears expired for stored backend", url); + } else { + console.error("[ctxce] Stored auth session appears expired for", url); + } if (rawExpires) { console.error("[ctxce] Session expired at", rawExpires); } @@ -169,6 +225,9 @@ async function doStatus(args) { outputJsonStatus(url, "ok", entry, rawExpires); return; } + if (usedFallback) { + console.error("[ctxce] Using stored backend for status:", url); + } console.error("[ctxce] Logged in to", url, "as", entry.userId || ""); if (rawExpires) { console.error("[ctxce] Session expires at", rawExpires); @@ -177,7 +236,26 @@ async function doStatus(args) { async function doLogout(args) { const { backendUrl } = parseAuthArgs(args); - const url = requireBackendUrl(backendUrl); + let url = getBackendUrl(backendUrl); + if (!url) { + // Fallback: use any stored auth entry when no backend is provided + const any = loadAnyAuthEntry(); + if (any && any.backendUrl) { + url = any.backendUrl; + console.error("[ctxce] Using stored backend for logout:", url); + } + } + if (!url) { + // Final fallback: default upload endpoint + url = getDefaultUploadBackend(); + if (url) { + console.error("[ctxce] Using default upload backend for logout:", url); + } + } + if (!url) { + console.error("[ctxce] Auth backend URL not configured and no stored sessions found. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + process.exit(1); + } const entry = loadAuthEntry(url); if (!entry) { console.error("[ctxce] No stored auth session for", url); diff --git a/scripts/auth_backend.py b/scripts/auth_backend.py index 52750ebe..7178c2ae 100644 --- a/scripts/auth_backend.py +++ b/scripts/auth_backend.py @@ -42,6 +42,12 @@ _default_auth_db_path = os.path.join(WORK_DIR, ".codebase", "ctxce_auth.sqlite") AUTH_DB_URL = os.environ.get("CTXCE_AUTH_DB_URL") or f"sqlite:///{_default_auth_db_path}" AUTH_SHARED_TOKEN = os.environ.get("CTXCE_AUTH_SHARED_TOKEN") +ALLOW_OPEN_TOKEN_LOGIN = ( + str(os.environ.get("CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN", "0")) + .strip() + .lower() + in {"1", "true", "yes", "on"} +) _SESSION_TTL_SECONDS_DEFAULT = 0 try: @@ -230,8 +236,17 @@ def create_session_for_token( if not AUTH_ENABLED: raise AuthDisabledError("Auth not enabled") if AUTH_SHARED_TOKEN: + # When a shared token is configured, require it for all token-based sessions. if not token or token != AUTH_SHARED_TOKEN: raise AuthInvalidToken("Invalid auth token") + else: + # Harden default behavior: when auth is enabled but no shared token is configured, + # disable token-based login unless explicitly allowed via env. + if not ALLOW_OPEN_TOKEN_LOGIN: + raise AuthInvalidToken( + "Token-based login disabled (no shared token configured; set CTXCE_AUTH_SHARED_TOKEN " + "or CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=1 to enable)" + ) user_id = client or "ctxce" meta: Dict[str, Any] = {} if workspace: diff --git a/vscode-extension/context-engine-uploader/README.md b/vscode-extension/context-engine-uploader/README.md index a0d7f02b..75494645 100644 --- a/vscode-extension/context-engine-uploader/README.md +++ b/vscode-extension/context-engine-uploader/README.md @@ -92,14 +92,21 @@ High-level steps: - Set `CTXCE_AUTH_ENABLED=1` in the upload/indexer environment. - Optionally set `CTXCE_AUTH_SHARED_TOKEN` for token-based login. - Optional: set `CTXCE_AUTH_ADMIN_TOKEN` for creating additional users via `/auth/users`. + - Optional (dev-only): set `CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=1` **only** if you want `/auth/login` + to issue sessions even when `CTXCE_AUTH_SHARED_TOKEN` is unset. By default (`0`/unset), + token-based login is disabled when no shared token is configured. - Point the bridge at the auth backend: - - In your local shell (where you run the `ctxce` CLI via `npx`), set `CTXCE_AUTH_BACKEND_URL` - to the upload service URL (e.g. `http://localhost:8004`). + - In your local shell (where you run the `ctxce` CLI via `npx`), you can either: + - Explicitly set `CTXCE_AUTH_BACKEND_URL` to the upload service URL (e.g. `http://localhost:8004`), or + - Let the CLI discover the backend URL in this order when you run `ctxce auth login`: + `--backend-url` / `--auth-url` → `CTXCE_AUTH_BACKEND_URL` → any stored auth entry → + the uploader's `upload_endpoint` (`CTXCE_UPLOAD_ENDPOINT` / `UPLOAD_ENDPOINT`) → + `http://localhost:8004`. Token-based login: ```bash -export CTXCE_AUTH_BACKEND_URL=http://localhost:8004 +export CTXCE_AUTH_BACKEND_URL=http://localhost:8004 # optional when using this extension export CTXCE_AUTH_TOKEN=change-me-dev-token # must match CTXCE_AUTH_SHARED_TOKEN in the stack # Obtain a session and cache it under ~/.ctxce/auth.json @@ -124,8 +131,6 @@ Username/password login (when you have real users): - Then login via the bridge: ```bash -export CTXCE_AUTH_BACKEND_URL=http://localhost:8004 - npx @context-engine-bridge/context-engine-mcp-bridge ctxce auth login \ --username you@example.com \ --password 'your-password' diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index eea96af5..4ef1e562 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -979,7 +979,17 @@ async function enhanceSelectionWithUnicorn() { } try { const cfg = vscode.workspace.getConfiguration('contextEngineUploader'); - const idxUrlRaw = (cfg.get('ctxIndexerUrl') || cfg.get('mcpIndexerUrl') || '').trim(); + const transportModeRaw = cfg.get('mcpTransportMode') || 'sse-remote'; + const serverModeRaw = cfg.get('mcpServerMode') || 'bridge'; + const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; + const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + let idxUrlRaw = (cfg.get('ctxIndexerUrl') || cfg.get('mcpIndexerUrl') || '').trim(); + if (serverMode === 'bridge' && transportMode === 'http') { + const bridgeUrl = resolveBridgeHttpUrl(); + if (bridgeUrl) { + idxUrlRaw = bridgeUrl; + } + } if (idxUrlRaw) { env.MCP_INDEXER_URL = idxUrlRaw; } @@ -1727,9 +1737,19 @@ async function scaffoldCtxConfigFiles(workspaceDir, collectionName) { // Ensure MCP_INDEXER_URL is present based on extension setting (for ctx.py) if (uploaderSettings) { try { - const ctxIndexerUrl = (uploaderSettings.get('ctxIndexerUrl') || 'http://localhost:8003/mcp').trim(); - if (ctxIndexerUrl) { - upsertEnv('MCP_INDEXER_URL', ctxIndexerUrl, { treatEmptyAsUnset: true }); + const transportModeRaw = uploaderSettings.get('mcpTransportMode') || 'sse-remote'; + const serverModeRaw = uploaderSettings.get('mcpServerMode') || 'bridge'; + const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; + const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + let targetUrl = (uploaderSettings.get('ctxIndexerUrl') || 'http://localhost:8003/mcp').trim(); + if (serverMode === 'bridge' && transportMode === 'http') { + const bridgeUrl = resolveBridgeHttpUrl(); + if (bridgeUrl) { + targetUrl = bridgeUrl; + } + } + if (targetUrl) { + upsertEnv('MCP_INDEXER_URL', targetUrl, { treatEmptyAsUnset: true }); } } catch (error) { log(`Failed to read ctxIndexerUrl setting for MCP_INDEXER_URL: ${error instanceof Error ? error.message : String(error)}`); From cc51799a7af581a03f8d5df0e63b920f108ec4f7 Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 17:50:10 +0000 Subject: [PATCH 32/55] feat(mcp-bridge): Propagate session ID to MCP indexer server --- ctx-mcp-bridge/src/mcpServer.js | 34 ++++++++++++++++++++++++--------- scripts/mcp_indexer_server.py | 15 +++++++++++++++ scripts/mcp_memory_server.py | 2 ++ 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/ctx-mcp-bridge/src/mcpServer.js b/ctx-mcp-bridge/src/mcpServer.js index fa2a5719..e57c633c 100644 --- a/ctx-mcp-bridge/src/mcpServer.js +++ b/ctx-mcp-bridge/src/mcpServer.js @@ -286,18 +286,20 @@ async function createBridgeServer(options) { const authBackendUrl = process.env.CTXCE_AUTH_BACKEND_URL || ""; let sessionId = explicitSession; - if (!sessionId) { + function resolveSessionId() { + const explicit = process.env.CTXCE_SESSION_ID || ""; + if (explicit) { + return explicit; + } let backendToUse = authBackendUrl; let entry = null; - if (backendToUse) { try { entry = loadAuthEntry(backendToUse); - } catch (err) { + } catch { entry = null; } } - if (!entry) { try { const any = loadAnyAuthEntry(); @@ -305,11 +307,10 @@ async function createBridgeServer(options) { backendToUse = any.backendUrl; entry = any.entry; } - } catch (err) { + } catch { entry = null; } } - if (entry) { let expired = false; const rawExpires = entry.expiresAt; @@ -320,11 +321,17 @@ async function createBridgeServer(options) { } } if (!expired && typeof entry.sessionId === "string" && entry.sessionId) { - sessionId = entry.sessionId; - } else if (expired) { + return entry.sessionId; + } + if (expired) { debugLog("[ctxce] Stored auth session appears expired; please run `ctxce auth login` again."); } } + return ""; + } + + if (!sessionId) { + sessionId = resolveSessionId(); } if (!sessionId) { @@ -477,7 +484,16 @@ async function createBridgeServer(options) { debugLog(`[ctxce] tools/call: ${name || ""}`); - // Attach session id so the target server can apply per-session defaults. + // Refresh session before each call; re-init clients if session changes. + const freshSession = resolveSessionId() || sessionId; + if (freshSession && freshSession !== sessionId) { + sessionId = freshSession; + try { + await initializeRemoteClients(true); + } catch (err) { + debugLog("[ctxce] Failed to reinitialize clients after session refresh: " + String(err)); + } + } if (sessionId && (args === undefined || args === null || typeof args === "object")) { const obj = args && typeof args === "object" ? { ...args } : {}; if (!Object.prototype.hasOwnProperty.call(obj, "session")) { diff --git a/scripts/mcp_indexer_server.py b/scripts/mcp_indexer_server.py index 50048e06..8c7fa398 100644 --- a/scripts/mcp_indexer_server.py +++ b/scripts/mcp_indexer_server.py @@ -3018,6 +3018,7 @@ async def search_tests_for( context_lines: Any = None, under: Any = None, language: Any = None, + session: Any = None, compact: Any = None, kwargs: Any = None, ) -> Dict[str, Any]: @@ -3056,6 +3057,7 @@ async def search_tests_for( under=under, language=language, path_glob=globs, + session=session, compact=compact, kwargs={k: v for k, v in _kwargs.items() if k not in {"path_glob"}}, ) @@ -3068,6 +3070,7 @@ async def search_config_for( include_snippet: Any = None, context_lines: Any = None, under: Any = None, + session: Any = None, compact: Any = None, kwargs: Any = None, ) -> Dict[str, Any]: @@ -3109,6 +3112,7 @@ async def search_config_for( include_snippet=include_snippet, context_lines=context_lines, under=under, + session=session, path_glob=globs, compact=compact, kwargs={k: v for k, v in _kwargs.items() if k not in {"path_glob"}}, @@ -3120,6 +3124,7 @@ async def search_callers_for( query: Any = None, limit: Any = None, language: Any = None, + session: Any = None, kwargs: Any = None, ) -> Dict[str, Any]: """Heuristic search for callers/usages of a symbol. @@ -3135,6 +3140,7 @@ async def search_callers_for( query=query, limit=limit, language=language, + session=session, kwargs=kwargs, ) @@ -3144,6 +3150,7 @@ async def search_importers_for( query: Any = None, limit: Any = None, language: Any = None, + session: Any = None, kwargs: Any = None, ) -> Dict[str, Any]: """Find files likely importing or referencing a module/symbol. @@ -3187,6 +3194,7 @@ async def search_importers_for( limit=limit, language=language, path_glob=globs, + session=session, kwargs={k: v for k, v in _kwargs.items() if k not in {"path_glob"}}, ) @@ -3657,6 +3665,7 @@ async def context_search( ext: Any = None, not_: Any = None, case: Any = None, + session: Any = None, compact: Any = None, # Repo scoping (cross-codebase isolation) repo: Any = None, # str, list[str], or "*" to search all repos @@ -4193,6 +4202,7 @@ def _maybe_dict(val: Any) -> Dict[str, Any]: case=case, compact=False, repo=repo, # Cross-codebase isolation + session=session, ) # Optional debug @@ -7926,6 +7936,7 @@ async def code_search( ext: Any = None, not_: Any = None, case: Any = None, + session: Any = None, compact: Any = None, kwargs: Any = None, ) -> Dict[str, Any]: @@ -7956,6 +7967,7 @@ async def code_search( ext=ext, not_=not_, case=case, + session=session, compact=compact, kwargs=kwargs, ) @@ -8108,6 +8120,8 @@ async def info_request( include_explanation: bool = None, # Relationship mapping include_relationships: bool = None, + # Auth/session (passed through to repo_search) + session: str = None, # Optional filters (pass-through to repo_search) limit: int = None, language: str = None, @@ -8185,6 +8199,7 @@ async def info_request( query=query, limit=eff_limit, per_path=3, # Better default for info requests + session=session, include_snippet=eff_snippet, context_lines=eff_context, language=language, diff --git a/scripts/mcp_memory_server.py b/scripts/mcp_memory_server.py index 446935bf..1436e205 100644 --- a/scripts/mcp_memory_server.py +++ b/scripts/mcp_memory_server.py @@ -409,6 +409,8 @@ def find( Cold-start option: set MEMORY_COLD_SKIP_DENSE=1 to skip dense embedding until the model is cached (useful on slow storage). """ + # _require_auth_session(session) # TODO: + coll = _resolve_collection(collection, session=session, ctx=ctx, extra_kwargs=kwargs) _ensure_once(coll) From d16d1c6c27d247ba7ea4f9b15cdcd631b64a903c Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 17:50:10 +0000 Subject: [PATCH 33/55] feat(ctx): Add GLM runtime support for plan generation and improve error handling --- scripts/ctx.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/scripts/ctx.py b/scripts/ctx.py index b1cc3c00..2c570a28 100755 --- a/scripts/ctx.py +++ b/scripts/ctx.py @@ -276,6 +276,30 @@ def parse_mcp_response(result: Dict[str, Any]) -> Optional[Dict[str, Any]]: return {"raw": text} +def _extract_tool_error(result: Dict[str, Any]) -> Optional[str]: + """Best-effort extraction of a tool-level error message from MCP response. + + Handles FastMCP-style isError + content.text as well as structuredContent.result.error. + Returns a human-readable error string when present, or None when no error detected. + """ + try: + res = result.get("result") or {} + if isinstance(res, dict) and res.get("isError") is True: + content = res.get("content") or [] + if isinstance(content, list) and content and isinstance(content[0], dict): + text = content[0].get("text") + if isinstance(text, str) and text.strip(): + return text.strip() + sc = res.get("structuredContent") or {} + rs = sc.get("result") or {} + err = rs.get("error") + if isinstance(err, str) and err.strip(): + return err.strip() + except Exception: + return None + return None + + def _compress_snippet(snippet: str, max_lines: int = 6) -> str: """Compact, high-signal subset of a code snippet. @@ -704,6 +728,40 @@ def _generate_plan(enhanced_prompt: str, context: str, note: str) -> str: "<|start_of_role|>assistant<|end_of_role|>" ) + runtime_kind = str(os.environ.get("REFRAG_RUNTIME", "llamacpp")).strip().lower() + + # GLM path mirrors rewrite_prompt behavior + if runtime_kind == "glm": + try: + from refrag_glm import GLMRefragClient # type: ignore + + client = GLMRefragClient() + response = client.client.chat.completions.create( + model=os.environ.get("GLM_MODEL", "glm-4.6"), + messages=[ + {"role": "system", "content": system_msg}, + {"role": "user", "content": user_msg}, + ], + max_tokens=200, + temperature=0.3, + stream=False, + ) + plan = ( + (response.choices[0].message.content if response and response.choices else "") + or "" + ).strip() + if not plan: + # Fall through to llama.cpp path + runtime_kind = "llamacpp" + else: + if "EXECUTION PLAN" not in plan.upper(): + plan = "EXECUTION PLAN:\n" + plan + return plan + except Exception as e: + sys.stderr.write(f"[DEBUG] Plan generation (GLM) failed, falling back to llama.cpp: {type(e).__name__}: {e}\n") + sys.stderr.flush() + runtime_kind = "llamacpp" + decoder_url = DECODER_URL # Safety: restrict to local decoder hosts parsed = urlparse(decoder_url) @@ -987,6 +1045,14 @@ def fetch_context(query: str, **filters) -> Tuple[str, str]: sys.stderr.flush() return "", f"Context retrieval failed: {error_msg}" + # Surface tool-level errors (including auth failures) explicitly instead of + # silently treating them as "no context". + tool_err = _extract_tool_error(result) + if tool_err: + sys.stderr.write(f"[DEBUG] repo_search tool error: {tool_err}\n") + sys.stderr.flush() + return "", f"Context retrieval failed: {tool_err}" + data = parse_mcp_response(result) if not data: sys.stderr.write("[DEBUG] repo_search returned no data\n") From e4fc9a23ed795526b6176b64fc3298bf99ba8bf7 Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 17:50:10 +0000 Subject: [PATCH 34/55] fix(mcp_router): Tolerate missing session id header in MCP handshake --- scripts/mcp_router.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/mcp_router.py b/scripts/mcp_router.py index 319a4420..e016ad15 100644 --- a/scripts/mcp_router.py +++ b/scripts/mcp_router.py @@ -564,9 +564,11 @@ def _mcp_handshake(base_url: str, timeout: float = 30.0) -> Dict[str, str]: sid = j.get("sessionId") except Exception: sid = None - if not sid: - raise RuntimeError("MCP handshake failed: no session id") - headers["Mcp-Session-Id"] = sid + # Tolerate servers (e.g., streamable-http bridge) that do not emit a session id header. + # In that case, proceed without attaching Mcp-Session-Id; downstream calls will still work + # for bridges that manage their own session lifecycle. + if sid: + headers["Mcp-Session-Id"] = sid # Send initialized notification (no id required) try: _post_raw_retry(base_url, {"jsonrpc": "2.0", "method": "notifications/initialized"}, headers, timeout=timeout) From c53b51bce4163d0613cadc2a7a5f7ef7eeedf9b1 Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 17:50:10 +0000 Subject: [PATCH 35/55] fix(upload_client): Special-case 401 errors in upload clients --- scripts/remote_upload_client.py | 7 +++++++ scripts/standalone_upload_client.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/scripts/remote_upload_client.py b/scripts/remote_upload_client.py index e65b69ed..82348799 100644 --- a/scripts/remote_upload_client.py +++ b/scripts/remote_upload_client.py @@ -955,6 +955,13 @@ def upload_bundle(self, bundle_path: str, manifest: Dict[str, Any]) -> Dict[str, error_msg += f": {response.text[:200]}" error_code = "HTTP_ERROR" + # Special-case 401 to make auth issues obvious to users + if response.status_code == 401: + if error_code in {None, "HTTP_ERROR"}: + error_code = "UNAUTHORIZED" + # Always append a clear hint for auth failures + error_msg += " (unauthorized; please log in with `ctxce auth login` and retry)" + last_error = {"success": False, "error": {"code": error_code, "message": error_msg, "status_code": response.status_code}} # Don't retry on client errors (except 429) diff --git a/scripts/standalone_upload_client.py b/scripts/standalone_upload_client.py index 47b6d810..a975d68a 100644 --- a/scripts/standalone_upload_client.py +++ b/scripts/standalone_upload_client.py @@ -1119,6 +1119,13 @@ def upload_bundle(self, bundle_path: str, manifest: Dict[str, Any]) -> Dict[str, error_msg += f": {response.text[:200]}" error_code = "HTTP_ERROR" + # Special-case 401 to make auth issues obvious to users + if response.status_code == 401: + if error_code in {None, "HTTP_ERROR"}: + error_code = "UNAUTHORIZED" + # Always append a clear hint for auth failures + error_msg += " (unauthorized; please log in with `ctxce auth login` and retry)" + last_error = {"success": False, "error": {"code": error_code, "message": error_msg, "status_code": response.status_code}} # Don't retry on client errors (except 429) From 18c48f764d52030141af1ffc895554d296250897 Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 17:50:11 +0000 Subject: [PATCH 36/55] fix(vscode-extension): Update dependencies in vscode extension build script --- vscode-extension/build/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vscode-extension/build/build.sh b/vscode-extension/build/build.sh index 60e0028c..856ea157 100755 --- a/vscode-extension/build/build.sh +++ b/vscode-extension/build/build.sh @@ -70,9 +70,9 @@ if [[ "$BUNDLE_DEPS" == "--bundle-deps" ]]; then # On macOS, urllib3 v2 + system LibreSSL emits NotOpenSSLWarning; pin <2 there. if [[ "$(uname -s)" == "Darwin" ]]; then echo "Detected macOS; pinning urllib3<2 to avoid LibreSSL/OpenSSL warning." - "$PYTHON_BIN" -m pip install -t "$STAGE_DIR/python_libs" "urllib3<2" requests charset_normalizer + "$PYTHON_BIN" -m pip install -t "$STAGE_DIR/python_libs" "urllib3<2" requests charset_normalizer "openai>=1.0" else - "$PYTHON_BIN" -m pip install -t "$STAGE_DIR/python_libs" requests urllib3 charset_normalizer + "$PYTHON_BIN" -m pip install -t "$STAGE_DIR/python_libs" requests urllib3 charset_normalizer "openai>=1.0" fi fi From f6123ca8fe14489aad5c182e87c4aa4aba714a1d Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 17:52:19 +0000 Subject: [PATCH 37/55] MCP bridge: version bump --- ctx-mcp-bridge/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctx-mcp-bridge/package.json b/ctx-mcp-bridge/package.json index 0c971840..fc57e3d9 100644 --- a/ctx-mcp-bridge/package.json +++ b/ctx-mcp-bridge/package.json @@ -1,6 +1,6 @@ { "name": "@context-engine-bridge/context-engine-mcp-bridge", - "version": "0.0.6", + "version": "0.0.7", "description": "Context Engine MCP bridge (http/stdio proxy combining indexer + memory servers)", "bin": { "ctxce": "bin/ctxce.js", From 197e42fe0d4a9c56e517a0f6a50a64a1353ae94f Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 11 Dec 2025 18:21:01 +0000 Subject: [PATCH 38/55] fix(mcp-indexer): allow clearing session defaults via empty strings Treat empty strings for collection, mode, under, and language in set_session_defaults as explicit unsets rather than no-ops. Update both per-connection and token-scoped SESSION_DEFAULTS maps to remove those keys before applying any new defaults, so sticky language/under/mode filters can be cleared without reconnecting. --- scripts/mcp_indexer_server.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/scripts/mcp_indexer_server.py b/scripts/mcp_indexer_server.py index 8c7fa398..9ffb1858 100644 --- a/scripts/mcp_indexer_server.py +++ b/scripts/mcp_indexer_server.py @@ -1710,20 +1710,22 @@ async def set_session_defaults( pass defaults: Dict[str, Any] = {} - if isinstance(collection, str) and collection.strip(): - defaults["collection"] = str(collection).strip() - if isinstance(mode, str) and mode.strip(): - defaults["mode"] = str(mode).strip() - if isinstance(under, str) and under.strip(): - defaults["under"] = str(under).strip() - if isinstance(language, str) and language.strip(): - defaults["language"] = str(language).strip() + unset_keys: set[str] = set() + for _key, _val in (("collection", collection), ("mode", mode), ("under", under), ("language", language)): + if isinstance(_val, str): + _s = _val.strip() + if _s: + defaults[_key] = _s + else: + unset_keys.add(_key) # Per-connection storage (preferred) try: - if ctx is not None and getattr(ctx, "session", None) is not None and defaults: + if ctx is not None and getattr(ctx, "session", None) is not None and (defaults or unset_keys): with _SESSION_CTX_LOCK: existing2 = SESSION_DEFAULTS_BY_SESSION.get(ctx.session) or {} + for _k in unset_keys: + existing2.pop(_k, None) existing2.update(defaults) SESSION_DEFAULTS_BY_SESSION[ctx.session] = existing2 except Exception: @@ -1734,9 +1736,11 @@ async def set_session_defaults( if not sid: sid = uuid.uuid4().hex[:12] try: - if defaults: + if defaults or unset_keys: with _SESSION_LOCK: existing = SESSION_DEFAULTS.get(sid) or {} + for _k in unset_keys: + existing.pop(_k, None) existing.update(defaults) SESSION_DEFAULTS[sid] = existing except Exception: From ef30572f650a24b0f3d2fdcacf23eafcd61741ac Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 00:59:30 +0000 Subject: [PATCH 39/55] mcp bridge: Remaps tool result paths to workspace-relative Adds path remapping for tool results to use workspace-relative paths. This change introduces a new module to handle the remapping of file paths returned by tools, specifically `repo_search`, `context_search`, and `context_answer`. It converts absolute or container paths to relative paths within the workspace, improving usability and portability of results. It also introduces environment variables for diagnostics and path overriding. --- ctx-mcp-bridge/src/mcpServer.js | 29 +-- ctx-mcp-bridge/src/resultPathMapping.js | 266 ++++++++++++++++++++++++ 2 files changed, 281 insertions(+), 14 deletions(-) create mode 100644 ctx-mcp-bridge/src/resultPathMapping.js diff --git a/ctx-mcp-bridge/src/mcpServer.js b/ctx-mcp-bridge/src/mcpServer.js index e57c633c..0cc12a53 100644 --- a/ctx-mcp-bridge/src/mcpServer.js +++ b/ctx-mcp-bridge/src/mcpServer.js @@ -1,3 +1,17 @@ +import process from "node:process"; +import fs from "node:fs"; +import path from "node:path"; +import { execSync } from "node:child_process"; +import { createServer } from "node:http"; +import { Server } from "@modelcontextprotocol/sdk/server/index.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js"; +import { loadAnyAuthEntry, loadAuthEntry } from "./authConfig.js"; +import { maybeRemapToolResult } from "./resultPathMapping.js"; + function debugLog(message) { try { const text = typeof message === "string" ? message : String(message); @@ -236,19 +250,6 @@ function isTransientToolError(error) { // Acts as a low-level proxy for tools, forwarding tools/list and tools/call // to the remote qdrant-indexer MCP server while adding a local `ping` tool. -import process from "node:process"; -import fs from "node:fs"; -import path from "node:path"; -import { execSync } from "node:child_process"; -import { createServer } from "node:http"; -import { Server } from "@modelcontextprotocol/sdk/server/index.js"; -import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; -import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; -import { Client } from "@modelcontextprotocol/sdk/client/index.js"; -import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; -import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js"; -import { loadAnyAuthEntry, loadAuthEntry } from "./authConfig.js"; - async function createBridgeServer(options) { const workspace = options.workspace || process.cwd(); const indexerUrl = options.indexerUrl; @@ -541,7 +542,7 @@ async function createBridgeServer(options) { undefined, { timeout: timeoutMs }, ); - return result; + return maybeRemapToolResult(name, result, workspace); } catch (err) { lastError = err; diff --git a/ctx-mcp-bridge/src/resultPathMapping.js b/ctx-mcp-bridge/src/resultPathMapping.js new file mode 100644 index 00000000..42f2972e --- /dev/null +++ b/ctx-mcp-bridge/src/resultPathMapping.js @@ -0,0 +1,266 @@ +import process from "node:process"; +import fs from "node:fs"; +import path from "node:path"; + +function envTruthy(value, defaultVal = false) { + try { + if (value === undefined || value === null) { + return defaultVal; + } + const s = String(value).trim().toLowerCase(); + if (!s) { + return defaultVal; + } + return s === "1" || s === "true" || s === "yes" || s === "on"; + } catch { + return defaultVal; + } +} + +function _posixToNative(rel) { + try { + if (!rel) { + return ""; + } + return String(rel).split("/").join(path.sep); + } catch { + return rel; + } +} + +function computeWorkspaceRelativePath(containerPath, hostPath) { + try { + const cont = typeof containerPath === "string" ? containerPath.trim() : ""; + if (cont.startsWith("/work/")) { + const rest = cont.slice("/work/".length); + const parts = rest.split("/").filter(Boolean); + if (parts.length >= 2) { + return parts.slice(1).join("/"); + } + if (parts.length === 1) { + return parts[0]; + } + } + } catch { + } + try { + const hp = typeof hostPath === "string" ? hostPath.trim() : ""; + if (!hp) { + return ""; + } + // If we don't have a container path, at least try to return a basename. + return path.posix.basename(hp.replace(/\\/g, "/")); + } catch { + return ""; + } +} + +function remapHitPaths(hit, workspaceRoot) { + if (!hit || typeof hit !== "object") { + return hit; + } + const hostPath = typeof hit.host_path === "string" ? hit.host_path : ""; + const containerPath = typeof hit.container_path === "string" ? hit.container_path : ""; + const relPath = computeWorkspaceRelativePath(containerPath, hostPath); + const out = { ...hit }; + if (relPath) { + out.rel_path = relPath; + } + if (workspaceRoot && relPath) { + try { + const relNative = _posixToNative(relPath); + const candidate = path.join(workspaceRoot, relNative); + const diagnostics = envTruthy(process.env.CTXCE_BRIDGE_PATH_DIAGNOSTICS, false); + const strictClientPath = envTruthy(process.env.CTXCE_BRIDGE_CLIENT_PATH_STRICT, false); + if (strictClientPath) { + out.client_path = candidate; + if (diagnostics) { + out.client_path_joined = candidate; + out.client_path_source = "workspace_join"; + } + } else { + // Prefer a host_path that is within the current bridge workspace. + // This keeps provenance (host_path) intact while providing a user-local + // absolute path even when the bridge workspace is a parent directory. + const hp = typeof hostPath === "string" ? hostPath : ""; + const hpNorm = hp ? hp.replace(/\\/g, path.sep) : ""; + if ( + hpNorm && + hpNorm.startsWith(workspaceRoot) && + (!fs.existsSync(candidate) || fs.existsSync(hpNorm)) + ) { + out.client_path = hpNorm; + if (diagnostics) { + out.client_path_joined = candidate; + out.client_path_source = "host_path"; + } + } else { + out.client_path = candidate; + if (diagnostics) { + out.client_path_joined = candidate; + out.client_path_source = "workspace_join"; + } + } + } + } catch { + // ignore + } + } + const overridePath = envTruthy(process.env.CTXCE_BRIDGE_OVERRIDE_PATH, true); + if (overridePath && relPath) { + out.path = relPath; + } + return out; +} + +function remapStringPath(p) { + try { + const s = typeof p === "string" ? p : ""; + if (!s) { + return p; + } + if (s.startsWith("/work/")) { + const rest = s.slice("/work/".length); + const parts = rest.split("/").filter(Boolean); + if (parts.length >= 2) { + const rel = parts.slice(1).join("/"); + const override = envTruthy(process.env.CTXCE_BRIDGE_OVERRIDE_PATH, true); + if (override) { + return rel; + } + return p; + } + } + return p; + } catch { + return p; + } +} + +function maybeParseToolJson(result) { + try { + if ( + result && + typeof result === "object" && + result.structuredContent && + typeof result.structuredContent === "object" + ) { + return { mode: "structured", value: result.structuredContent }; + } + } catch { + } + try { + const content = result && result.content; + if (!Array.isArray(content)) { + return null; + } + const first = content.find( + (c) => c && c.type === "text" && typeof c.text === "string", + ); + if (!first) { + return null; + } + const txt = String(first.text || "").trim(); + if (!txt || !(txt.startsWith("{") || txt.startsWith("["))) { + return null; + } + return { mode: "text", value: JSON.parse(txt) }; + } catch { + return null; + } +} + +function applyPathMappingToPayload(payload, workspaceRoot) { + if (!payload || typeof payload !== "object") { + return payload; + } + const out = Array.isArray(payload) ? payload.slice() : { ...payload }; + + const mapHitsArray = (arr) => { + if (!Array.isArray(arr)) { + return arr; + } + return arr.map((h) => remapHitPaths(h, workspaceRoot)); + }; + + // Common result shapes across tools + if (Array.isArray(out.results)) { + out.results = mapHitsArray(out.results); + } + if (Array.isArray(out.citations)) { + out.citations = mapHitsArray(out.citations); + } + if (Array.isArray(out.related_paths)) { + out.related_paths = out.related_paths.map((p) => remapStringPath(p)); + } + + // context_search: {results:[{source:"code"|"memory", ...}]} + if (Array.isArray(out.results)) { + out.results = out.results.map((r) => { + if (!r || typeof r !== "object") { + return r; + } + // Only code results have path-like fields + return remapHitPaths(r, workspaceRoot); + }); + } + + // Some tools nest under {result:{...}} + if (out.result && typeof out.result === "object") { + out.result = applyPathMappingToPayload(out.result, workspaceRoot); + } + + return out; +} + +export function maybeRemapToolResult(name, result, workspaceRoot) { + try { + if (!name || !result || !workspaceRoot) { + return result; + } + const enabled = envTruthy(process.env.CTXCE_BRIDGE_MAP_PATHS, true); + if (!enabled) { + return result; + } + const lower = String(name).toLowerCase(); + const shouldMap = ( + lower === "repo_search" || + lower === "context_search" || + lower === "context_answer" || + lower.endsWith("search_tests_for") || + lower.endsWith("search_config_for") || + lower.endsWith("search_callers_for") || + lower.endsWith("search_importers_for") + ); + if (!shouldMap) { + return result; + } + + const parsed = maybeParseToolJson(result); + if (!parsed) { + return result; + } + + const mapped = applyPathMappingToPayload(parsed.value, workspaceRoot); + if (parsed.mode === "structured") { + return { ...result, structuredContent: mapped }; + } + + // Replace text payload for clients that only read `content[].text` + try { + const content = Array.isArray(result.content) ? result.content.slice() : []; + const idx = content.findIndex( + (c) => c && c.type === "text" && typeof c.text === "string", + ); + if (idx >= 0) { + content[idx] = { ...content[idx], text: JSON.stringify(mapped) }; + return { ...result, content }; + } + } catch { + // ignore + } + return result; + } catch { + return result; + } +} From 27189cb31fb79224ec874671cc5974cc7a153afc Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 13:38:42 +0000 Subject: [PATCH 40/55] feat: implement collection registry and ACL --- .env.example | 14 ++ docker-compose.dev-remote.yml | 15 ++ scripts/auth_backend.py | 297 +++++++++++++++++++++++++++++++++- scripts/health_check.py | 11 ++ scripts/mcp_auth.py | 61 +++++++ scripts/mcp_indexer_server.py | 29 ++-- scripts/mcp_memory_server.py | 5 + scripts/upload_service.py | 4 + 8 files changed, 416 insertions(+), 20 deletions(-) create mode 100644 scripts/mcp_auth.py diff --git a/.env.example b/.env.example index 15f927eb..ced87907 100644 --- a/.env.example +++ b/.env.example @@ -270,6 +270,20 @@ INFO_REQUEST_CONTEXT_LINES=5 # active sessions are extended with a sliding window whenever they are used. # CTXCE_AUTH_SESSION_TTL_SECONDS=0 +# Collection registry & ACL (only used when CTXCE_AUTH_ENABLED=1). +# When enabled, infrastructure/health checks may populate an internal SQLite +# registry of known Qdrant collections, and ACL rules can be applied by services. + +# ACL bypass (dev/early deployments): allow all users to access all collections. +# CTXCE_ACL_ALLOW_ALL=0 + +# MCP boundary enforcement (OFF by default). +# When enabled, MCP servers will enforce collection ACLs using the auth DB. +# Requires CTXCE_AUTH_ENABLED=1. If CTXCE_ACL_ALLOW_ALL=1, enforcement is bypassed. +# This is intended for gradually rolling out collection-level permissions without +# changing existing client auth/session mechanisms. +# CTXCE_MCP_ACL_ENFORCE=0 + # Bridge-side configuration (ctx-mcp-bridge): # The bridge will POST to this URL for auth login and store the returned # session id, then inject it into all MCP tool calls as the `session` field. diff --git a/docker-compose.dev-remote.yml b/docker-compose.dev-remote.yml index 8efac541..ef80cf3b 100644 --- a/docker-compose.dev-remote.yml +++ b/docker-compose.dev-remote.yml @@ -35,10 +35,13 @@ services: - QDRANT_URL=${QDRANT_URL} # Optional auth configuration (fully opt-in via .env) - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} - COLLECTION_NAME=${COLLECTION_NAME} - PATH_EMIT_MODE=auto - HF_HOME=/work/.cache/huggingface @@ -83,10 +86,13 @@ services: - QDRANT_URL=${QDRANT_URL} # Optional auth configuration (fully opt-in via .env) - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} - REFRAG_DECODER=${REFRAG_DECODER:-1} - REFRAG_RUNTIME=${REFRAG_RUNTIME:-llamacpp} - GLM_API_KEY=${GLM_API_KEY} @@ -134,10 +140,13 @@ services: - QDRANT_URL=${QDRANT_URL} # Optional auth configuration (fully opt-in via .env) - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} - COLLECTION_NAME=${COLLECTION_NAME} - PATH_EMIT_MODE=auto - HF_HOME=/work/.cache/huggingface @@ -182,10 +191,13 @@ services: - QDRANT_URL=${QDRANT_URL} # Optional auth configuration (fully opt-in via .env) - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} - REFRAG_DECODER=${REFRAG_DECODER:-1} - REFRAG_RUNTIME=${REFRAG_RUNTIME:-llamacpp} - GLM_API_KEY=${GLM_API_KEY} @@ -360,10 +372,13 @@ services: - UPLOAD_TIMEOUT_SECS=300 # Optional auth configuration (fully opt-in via .env) - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} # Indexing configuration - COLLECTION_NAME=${COLLECTION_NAME} diff --git a/scripts/auth_backend.py b/scripts/auth_backend.py index 7178c2ae..cb707380 100644 --- a/scripts/auth_backend.py +++ b/scripts/auth_backend.py @@ -42,6 +42,11 @@ _default_auth_db_path = os.path.join(WORK_DIR, ".codebase", "ctxce_auth.sqlite") AUTH_DB_URL = os.environ.get("CTXCE_AUTH_DB_URL") or f"sqlite:///{_default_auth_db_path}" AUTH_SHARED_TOKEN = os.environ.get("CTXCE_AUTH_SHARED_TOKEN") +COLLECTION_REGISTRY_ENABLED = AUTH_ENABLED +ACL_ALLOW_ALL = ( + str(os.environ.get("CTXCE_ACL_ALLOW_ALL", "0")).strip().lower() + in {"1", "true", "yes", "on"} +) ALLOW_OPEN_TOKEN_LOGIN = ( str(os.environ.get("CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN", "0")) .strip() @@ -101,8 +106,24 @@ def _ensure_auth_db() -> None: "CREATE TABLE IF NOT EXISTS sessions (id TEXT PRIMARY KEY, user_id TEXT, created_at INTEGER, expires_at INTEGER, metadata_json TEXT)" ) conn.execute( - "CREATE TABLE IF NOT EXISTS users (id TEXT PRIMARY KEY, username TEXT UNIQUE NOT NULL, password_hash TEXT NOT NULL, created_at INTEGER NOT NULL, metadata_json TEXT)" + "CREATE TABLE IF NOT EXISTS users (id TEXT PRIMARY KEY, username TEXT UNIQUE NOT NULL, password_hash TEXT NOT NULL, created_at INTEGER NOT NULL, metadata_json TEXT, role TEXT NOT NULL DEFAULT 'user')" + ) + conn.execute( + "CREATE TABLE IF NOT EXISTS collections (id TEXT PRIMARY KEY, qdrant_collection TEXT UNIQUE NOT NULL, created_at INTEGER NOT NULL, metadata_json TEXT, is_deleted INTEGER NOT NULL DEFAULT 0)" ) + conn.execute( + "CREATE TABLE IF NOT EXISTS collection_acl (collection_id TEXT NOT NULL, user_id TEXT NOT NULL, permission TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (collection_id, user_id))" + ) + try: + cur = conn.cursor() + cur.execute("PRAGMA table_info(users)") + cols = [r[1] for r in cur.fetchall() or []] + if "role" not in cols: + conn.execute( + "ALTER TABLE users ADD COLUMN role TEXT NOT NULL DEFAULT 'user'" + ) + except Exception: + pass def _hash_password(password: str) -> str: @@ -169,9 +190,28 @@ def _get_user_by_username(username: str) -> Optional[Dict[str, Any]]: "password_hash": row[2], "created_at": row[3], "metadata_json": row[4], + "role": row[5], } +def _get_user_role(user_id: str) -> Optional[str]: + uid = (user_id or "").strip() + if not uid: + return None + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute("SELECT role FROM users WHERE id = ?", (uid,)) + row = cur.fetchone() + if not row: + return None + return str(row[0] or "").strip() or None + + +def is_admin_user(user_id: str) -> bool: + return (_get_user_role(user_id) or "").lower() == "admin" + + def has_any_users() -> bool: """Return True if at least one user exists. @@ -310,3 +350,258 @@ def validate_session(session_id: str) -> Optional[Dict[str, Any]]: "expires_at": expires_ts, "metadata": meta or {}, } + + +def ensure_collection(qdrant_collection: str, metadata: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + if not COLLECTION_REGISTRY_ENABLED: + raise AuthDisabledError("Collection registry not enabled") + name = (qdrant_collection or "").strip() + if not name: + raise ValueError("qdrant_collection is required") + _ensure_db() + now_ts = int(datetime.now().timestamp()) + meta_json: Optional[str] = None + if metadata: + try: + meta_json = json.dumps(metadata) + except Exception: + meta_json = None + with _db_connection() as conn: + with conn: + cur = conn.cursor() + cur.execute( + "SELECT id, qdrant_collection, created_at, metadata_json, is_deleted FROM collections WHERE qdrant_collection = ?", + (name,), + ) + row = cur.fetchone() + if row: + return { + "id": row[0], + "qdrant_collection": row[1], + "created_at": int(row[2] or 0), + "metadata_json": row[3], + "is_deleted": int(row[4] or 0), + } + + coll_id = uuid.uuid4().hex + conn.execute( + "INSERT INTO collections (id, qdrant_collection, created_at, metadata_json, is_deleted) VALUES (?, ?, ?, ?, 0)", + (coll_id, name, now_ts, meta_json), + ) + return { + "id": coll_id, + "qdrant_collection": name, + "created_at": now_ts, + "metadata_json": meta_json, + "is_deleted": 0, + } + + +def ensure_collections(collections: List[str]) -> int: + if not COLLECTION_REGISTRY_ENABLED: + raise AuthDisabledError("Collection registry not enabled") + names = [str(c).strip() for c in (collections or []) if str(c).strip()] + if not names: + return 0 + _ensure_db() + before_count = 0 + after_count = 0 + failures: List[str] = [] + try: + with _db_connection() as conn: + cur = conn.cursor() + cur.execute("SELECT COUNT(1) FROM collections") + row = cur.fetchone() + if row: + before_count = int(row[0] or 0) + except Exception: + before_count = 0 + for name in names: + try: + ensure_collection(name) + except Exception as e: + failures.append(f"{name}: {e}") + continue + try: + with _db_connection() as conn: + cur = conn.cursor() + cur.execute("SELECT COUNT(1) FROM collections") + row = cur.fetchone() + if row: + after_count = int(row[0] or 0) + except Exception: + after_count = before_count + + delta = max(0, after_count - before_count) + if failures and len(failures) >= len(names) and delta == 0: + raise RuntimeError("Failed to sync collections registry: " + "; ".join(failures[:3])) + return delta + + +def grant_collection_access(user_id: str, qdrant_collection: str, permission: str = "read") -> Dict[str, Any]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + uid = (user_id or "").strip() + perm = (permission or "read").strip() or "read" + if not uid: + raise ValueError("user_id is required") + coll = ensure_collection(qdrant_collection) + now_ts = int(datetime.now().timestamp()) + with _db_connection() as conn: + with conn: + conn.execute( + "INSERT OR REPLACE INTO collection_acl (collection_id, user_id, permission, created_at) VALUES (?, ?, ?, ?)", + (coll.get("id"), uid, perm, now_ts), + ) + return {"collection_id": coll.get("id"), "user_id": uid, "permission": perm} + + +def revoke_collection_access(user_id: str, qdrant_collection: str) -> bool: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + uid = (user_id or "").strip() + name = (qdrant_collection or "").strip() + if not uid: + raise ValueError("user_id is required") + if not name: + raise ValueError("qdrant_collection is required") + with _db_connection() as conn: + cur = conn.cursor() + cur.execute( + "SELECT c.id FROM collections c WHERE c.qdrant_collection = ? AND c.is_deleted = 0", + (name,), + ) + row = cur.fetchone() + if not row: + return False + coll_id = row[0] + with conn: + cur.execute( + "DELETE FROM collection_acl WHERE collection_id = ? AND user_id = ?", + (coll_id, uid), + ) + return bool(cur.rowcount and cur.rowcount > 0) + + +def has_collection_access(user_id: str, qdrant_collection: str, permission: str = "read") -> bool: + if ACL_ALLOW_ALL: + return True + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + uid = (user_id or "").strip() + if not uid: + return False + if is_admin_user(uid): + return True + name = (qdrant_collection or "").strip() + if not name: + return False + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute( + "SELECT c.id FROM collections c WHERE c.qdrant_collection = ? AND c.is_deleted = 0", + (name,), + ) + row = cur.fetchone() + if not row: + return False + coll_id = row[0] + cur.execute( + "SELECT permission FROM collection_acl WHERE collection_id = ? AND user_id = ?", + (coll_id, uid), + ) + perm_row = cur.fetchone() + if not perm_row: + return False + granted = str(perm_row[0] or "").strip().lower() + want = (permission or "read").strip().lower() + if granted == "admin": + return True + if want == "read": + return granted in {"read", "write"} + if want == "write": + return granted == "write" + return granted == want + + +def list_users() -> List[Dict[str, Any]]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute("SELECT id, username, created_at, role FROM users ORDER BY created_at ASC") + rows = cur.fetchall() or [] + out: List[Dict[str, Any]] = [] + for r in rows: + out.append( + { + "id": r[0], + "username": r[1], + "created_at": int(r[2] or 0), + "role": r[3], + } + ) + return out + + +def list_collections(include_deleted: bool = False) -> List[Dict[str, Any]]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + if include_deleted: + cur.execute( + "SELECT id, qdrant_collection, created_at, is_deleted FROM collections ORDER BY created_at ASC" + ) + else: + cur.execute( + "SELECT id, qdrant_collection, created_at, is_deleted FROM collections WHERE is_deleted = 0 ORDER BY created_at ASC" + ) + rows = cur.fetchall() or [] + out: List[Dict[str, Any]] = [] + for r in rows: + out.append( + { + "id": r[0], + "qdrant_collection": r[1], + "created_at": int(r[2] or 0), + "is_deleted": int(r[3] or 0), + } + ) + return out + + +def list_collection_acl() -> List[Dict[str, Any]]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute( + "SELECT a.collection_id, c.qdrant_collection, a.user_id, u.username, u.role, a.permission, a.created_at " + "FROM collection_acl a " + "JOIN collections c ON c.id = a.collection_id " + "LEFT JOIN users u ON u.id = a.user_id " + "WHERE c.is_deleted = 0 " + "ORDER BY c.qdrant_collection ASC, u.username ASC" + ) + rows = cur.fetchall() or [] + out: List[Dict[str, Any]] = [] + for r in rows: + out.append( + { + "collection_id": r[0], + "qdrant_collection": r[1], + "user_id": r[2], + "username": r[3], + "user_role": r[4], + "permission": r[5], + "created_at": int(r[6] or 0), + } + ) + return out diff --git a/scripts/health_check.py b/scripts/health_check.py index 67e80e2e..574483cc 100644 --- a/scripts/health_check.py +++ b/scripts/health_check.py @@ -14,6 +14,7 @@ from scripts.utils import sanitize_vector_name +from scripts.auth_backend import ensure_collections, AuthDisabledError def assert_true(cond: bool, msg: str): @@ -44,6 +45,16 @@ def main(): collections_response = client.get_collections() collections = [c.name for c in collections_response.collections] print(f"Found collections: {collections}") + try: + created = ensure_collections(collections) + if created: + print(f"[OK] Synced collections registry (new entries: {created})") + else: + print("[OK] Synced collections registry") + except AuthDisabledError: + pass + except Exception as e: + print(f"[WARN] Failed to sync collections registry: {e}") except Exception as e: print(f"Error getting collections: {e}") sys.exit(1) diff --git a/scripts/mcp_auth.py b/scripts/mcp_auth.py new file mode 100644 index 00000000..06ccc0cb --- /dev/null +++ b/scripts/mcp_auth.py @@ -0,0 +1,61 @@ +import os +from typing import Any, Dict, Optional + +try: + from scripts.logger import ValidationError +except Exception: + + class ValidationError(Exception): + pass + + +try: + from scripts.auth_backend import ( + AUTH_ENABLED as AUTH_ENABLED_AUTH, + ACL_ALLOW_ALL as ACL_ALLOW_ALL_AUTH, + validate_session as _auth_validate_session, + has_collection_access as _has_collection_access, + ) +except Exception: + AUTH_ENABLED_AUTH = False + ACL_ALLOW_ALL_AUTH = True + + def _auth_validate_session(session_id: str): # type: ignore[no-redef] + return None + + def _has_collection_access( + user_id: str, qdrant_collection: str, permission: str = "read" + ) -> bool: # type: ignore[no-redef] + return True + + +ACL_ENFORCE = ( + str(os.environ.get("CTXCE_MCP_ACL_ENFORCE", "0")).strip().lower() + in {"1", "true", "yes", "on"} +) + + +def require_auth_session(session: Optional[str]) -> Optional[Dict[str, Any]]: + if not AUTH_ENABLED_AUTH: + return None + sid = (session or "").strip() + if not sid: + raise ValidationError("Missing session for authorized operation") + info = _auth_validate_session(sid) + if not info: + raise ValidationError("Invalid or expired session") + return info + + +def require_collection_access(user_id: Optional[str], collection: str, perm: str) -> None: + if not ACL_ENFORCE or not AUTH_ENABLED_AUTH: + return + if ACL_ALLOW_ALL_AUTH: + return + uid = (user_id or "").strip() + if not uid: + raise ValidationError("Not authorized: missing user id") + if not _has_collection_access(uid, collection, perm): + raise ValidationError( + f"Forbidden: {perm} access to collection '{collection}' denied" + ) diff --git a/scripts/mcp_indexer_server.py b/scripts/mcp_indexer_server.py index 9ffb1858..ce47d0c9 100644 --- a/scripts/mcp_indexer_server.py +++ b/scripts/mcp_indexer_server.py @@ -123,25 +123,10 @@ def safe_bool(value, default=False, logger=None, context=""): return default -try: - from scripts.auth_backend import AUTH_ENABLED as AUTH_ENABLED_AUTH, validate_session as _auth_validate_session -except Exception: - AUTH_ENABLED_AUTH = False - - def _auth_validate_session(session_id: str): # type: ignore[no-redef] - return None - - -def _require_auth_session(session: Optional[str]) -> Optional[Dict[str, Any]]: - if not AUTH_ENABLED_AUTH: - return None - sid = (session or "").strip() - if not sid: - raise ValidationError("Missing session for authorized operation") - info = _auth_validate_session(sid) - if not info: - raise ValidationError("Invalid or expired session") - return info +from scripts.mcp_auth import ( + require_auth_session as _require_auth_session, + require_collection_access as _require_collection_access, +) # Global lock to guard temporary env toggles used during ReFRAG retrieval/decoding @@ -1024,6 +1009,8 @@ async def qdrant_index_root( except Exception: coll = _default_collection() + _require_collection_access((sess or {}).get("user_id") if sess else None, coll, "write") + env = os.environ.copy() env["QDRANT_URL"] = QDRANT_URL env["COLLECTION_NAME"] = coll @@ -1652,6 +1639,8 @@ async def qdrant_index( except Exception: coll = _default_collection() + _require_collection_access((sess or {}).get("user_id") if sess else None, coll, "write") + env = os.environ.copy() env["QDRANT_URL"] = QDRANT_URL env["COLLECTION_NAME"] = coll @@ -2079,6 +2068,8 @@ def _to_str(x, default=""): env_fallback = (os.environ.get("DEFAULT_COLLECTION") or os.environ.get("COLLECTION_NAME") or "my-collection").strip() collection = coll_hint or env_fallback + _require_collection_access((sess or {}).get("user_id") if sess else None, collection, "read") + # Optional mode knob: "code_first" (default for IDE), "docs_first", "balanced" if not mode: mode = mode_hint diff --git a/scripts/mcp_memory_server.py b/scripts/mcp_memory_server.py index 1436e205..e976b399 100644 --- a/scripts/mcp_memory_server.py +++ b/scripts/mcp_memory_server.py @@ -13,6 +13,11 @@ from mcp.server.fastmcp import FastMCP # type: ignore Context = Any # type: ignore +from scripts.mcp_auth import ( + require_auth_session as _require_auth_session, + require_collection_access as _require_collection_access, +) + from qdrant_client import QdrantClient, models # Env diff --git a/scripts/upload_service.py b/scripts/upload_service.py index a5f70e7f..9bb2c44e 100644 --- a/scripts/upload_service.py +++ b/scripts/upload_service.py @@ -78,6 +78,10 @@ def logical_repo_reuse_enabled() -> bool: # type: ignore[no-redef] WORK_DIR = os.environ.get("WORK_DIR", "/work") MAX_BUNDLE_SIZE_MB = int(os.environ.get("MAX_BUNDLE_SIZE_MB", "100")) UPLOAD_TIMEOUT_SECS = int(os.environ.get("UPLOAD_TIMEOUT_SECS", "300")) +CTXCE_MCP_ACL_ENFORCE = ( + str(os.environ.get("CTXCE_MCP_ACL_ENFORCE", "0")).strip().lower() + in {"1", "true", "yes", "on"} +) # FastAPI app app = FastAPI( From 492010ef5caf2f5c606ccab20c4d20b78ae897ca Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 13:38:42 +0000 Subject: [PATCH 41/55] feat: add admin UI for user and collection ACL management --- requirements.txt | 1 + scripts/admin_ui.py | 78 ++++++++++ scripts/auth_backend.py | 36 +++-- scripts/upload_service.py | 274 ++++++++++++++++++++++++++++++++- templates/admin/acl.html | 110 +++++++++++++ templates/admin/base.html | 39 +++++ templates/admin/bootstrap.html | 27 ++++ templates/admin/error.html | 9 ++ templates/admin/login.html | 25 +++ 9 files changed, 587 insertions(+), 12 deletions(-) create mode 100644 scripts/admin_ui.py create mode 100644 templates/admin/acl.html create mode 100644 templates/admin/base.html create mode 100644 templates/admin/bootstrap.html create mode 100644 templates/admin/error.html create mode 100644 templates/admin/login.html diff --git a/requirements.txt b/requirements.txt index e901481c..3a2c0d43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ fastmcp==2.12.4 fastapi uvicorn[standard] python-multipart +jinja2 openai>=1.0 # Test-only diff --git a/scripts/admin_ui.py b/scripts/admin_ui.py new file mode 100644 index 00000000..8104f5a0 --- /dev/null +++ b/scripts/admin_ui.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Optional + +from fastapi import Request +from starlette.templating import Jinja2Templates +from jinja2 import select_autoescape + + +_TEMPLATES_DIR = Path(__file__).resolve().parent.parent / "templates" +_templates = Jinja2Templates(directory=str(_TEMPLATES_DIR)) +_templates.env.autoescape = select_autoescape(enabled_extensions=("html", "xml"), default=True) + + +def render_admin_login( + request: Request, + error: Optional[str] = None, + status_code: int = 200, +) -> Any: + return _templates.TemplateResponse( + "admin/login.html", + {"request": request, "title": "CTXCE Admin Login", "error": error}, + status_code=status_code, + ) + + +def render_admin_bootstrap( + request: Request, + error: Optional[str] = None, + status_code: int = 200, +) -> Any: + return _templates.TemplateResponse( + "admin/bootstrap.html", + {"request": request, "title": "CTXCE Admin Bootstrap", "error": error}, + status_code=status_code, + ) + + +def render_admin_acl( + request: Request, + users: Any, + collections: Any, + grants: Any, + status_code: int = 200, +) -> Any: + return _templates.TemplateResponse( + "admin/acl.html", + { + "request": request, + "title": "CTXCE Admin ACL", + "users": users, + "collections": collections, + "grants": grants, + }, + status_code=status_code, + ) + + +def render_admin_error( + request: Request, + title: str, + message: str, + back_href: str = "/admin", + status_code: int = 400, +) -> Any: + return _templates.TemplateResponse( + "admin/error.html", + { + "request": request, + "title": title, + "message": message, + "back_href": back_href, + }, + status_code=status_code, + ) diff --git a/scripts/auth_backend.py b/scripts/auth_backend.py index cb707380..f9e3fe6a 100644 --- a/scripts/auth_backend.py +++ b/scripts/auth_backend.py @@ -29,7 +29,7 @@ import uuid from contextlib import contextmanager from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, List # Configuration WORK_DIR = os.environ.get("WORK_DIR", "/work") @@ -92,7 +92,7 @@ def _db_connection(): conn.close() -def _ensure_auth_db() -> None: +def _ensure_db() -> None: path = _get_auth_db_path() if not path: return @@ -149,11 +149,14 @@ def _verify_password(password: str, encoded: str) -> bool: def create_user( - username: str, password: str, metadata: Optional[Dict[str, Any]] = None + username: str, + password: str, + metadata: Optional[Dict[str, Any]] = None, + role: Optional[str] = None, ) -> Dict[str, Any]: if not AUTH_ENABLED: raise AuthDisabledError("Auth not enabled") - _ensure_auth_db() + _ensure_db() path = _get_auth_db_path() now_ts = int(datetime.now().timestamp()) password_hash = _hash_password(password) @@ -166,19 +169,30 @@ def create_user( user_id = uuid.uuid4().hex with _db_connection() as conn: with conn: + desired_role = (str(role).strip().lower() if role is not None else "") + if desired_role and desired_role not in {"user", "admin"}: + raise ValueError("Invalid role") + role_val = desired_role or "user" + try: + cur = conn.cursor() + cur.execute("SELECT 1 FROM users LIMIT 1") + if not cur.fetchone(): + role_val = "admin" + except Exception: + role_val = "user" conn.execute( - "INSERT INTO users (id, username, password_hash, created_at, metadata_json) VALUES (?, ?, ?, ?, ?)", - (user_id, username, password_hash, now_ts, meta_json), + "INSERT INTO users (id, username, password_hash, created_at, metadata_json, role) VALUES (?, ?, ?, ?, ?, ?)", + (user_id, username, password_hash, now_ts, meta_json, role_val), ) - return {"user_id": user_id, "username": username} + return {"id": user_id, "user_id": user_id, "username": username, "role": role_val} def _get_user_by_username(username: str) -> Optional[Dict[str, Any]]: - _ensure_auth_db() + _ensure_db() with _db_connection() as conn: cur = conn.cursor() cur.execute( - "SELECT id, username, password_hash, created_at, metadata_json FROM users WHERE username = ?", + "SELECT id, username, password_hash, created_at, metadata_json, role FROM users WHERE username = ?", (username,), ) row = cur.fetchone() @@ -220,7 +234,7 @@ def has_any_users() -> bool: """ if not AUTH_ENABLED: raise AuthDisabledError("Auth not enabled") - _ensure_auth_db() + _ensure_db() with _db_connection() as conn: cur = conn.cursor() cur.execute("SELECT 1 FROM users LIMIT 1") @@ -306,7 +320,7 @@ def validate_session(session_id: str) -> Optional[Dict[str, Any]]: sid = (session_id or "").strip() if not sid: return None - _ensure_auth_db() + _ensure_db() with _db_connection() as conn: cur = conn.cursor() cur.execute( diff --git a/scripts/upload_service.py b/scripts/upload_service.py index 9bb2c44e..d5aee9b5 100644 --- a/scripts/upload_service.py +++ b/scripts/upload_service.py @@ -19,7 +19,7 @@ import uvicorn from fastapi import FastAPI, File, UploadFile, Form, HTTPException, Request, status -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, RedirectResponse from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel, Field from scripts.auth_backend import ( @@ -161,6 +161,75 @@ class PasswordLoginRequest(BaseModel): password: str workspace: Optional[str] = None + +ADMIN_SESSION_COOKIE_NAME = "ctxce_session" + + +def _get_session_candidate_from_request(request: Request) -> Dict[str, Any]: + sid = (request.cookies.get(ADMIN_SESSION_COOKIE_NAME) or "").strip() + if sid: + return {"session_id": sid, "source": "cookie"} + try: + qp = request.query_params + sid = (qp.get("session") or qp.get("session_id") or qp.get("sessionId") or "").strip() + except Exception: + sid = "" + if sid: + return {"session_id": sid, "source": "query"} + sid = ( + (request.headers.get("X-Session-Id") or "").strip() + or (request.headers.get("X-Auth-Session") or "").strip() + ) + if sid: + return {"session_id": sid, "source": "header"} + return {"session_id": "", "source": ""} + + +def _set_admin_session_cookie(resp: Any, session_id: str) -> Any: + sid = (session_id or "").strip() + if not sid: + return resp + try: + kwargs: Dict[str, Any] = { + "key": ADMIN_SESSION_COOKIE_NAME, + "value": sid, + "httponly": True, + "samesite": "lax", + "path": "/", + } + ttl = int(AUTH_SESSION_TTL_SECONDS or 0) + if ttl > 0: + kwargs["max_age"] = ttl + resp.set_cookie(**kwargs) + except Exception: + pass + return resp + + +def _get_valid_session_record(request: Request) -> Optional[Dict[str, Any]]: + sid = (_get_session_candidate_from_request(request).get("session_id") or "").strip() + if not sid: + return None + try: + return validate_session(sid) + except AuthDisabledError: + return None + except Exception as e: + logger.error(f"[upload_service] Failed to validate session cookie: {e}") + raise HTTPException(status_code=500, detail="Failed to validate auth session") + + +def _require_admin_session(request: Request) -> Dict[str, Any]: + if not AUTH_ENABLED: + raise HTTPException(status_code=404, detail="Auth disabled") + record = _get_valid_session_record(request) + if record is None: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") + user_id = str(record.get("user_id") or "").strip() + if not user_id or not is_admin_user(user_id): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin required") + return record + def get_workspace_key(workspace_path: str) -> str: """Generate 16-char hash for collision avoidance in remote uploads. @@ -524,6 +593,209 @@ async def auth_login(payload: AuthLoginRequest): ) +@app.get("/admin") +async def admin_root(request: Request): + if not AUTH_ENABLED: + raise HTTPException(status_code=404, detail="Auth disabled") + try: + users_exist = has_any_users() + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to inspect user state for admin UI: {e}") + raise HTTPException(status_code=500, detail="Failed to inspect user state") + + if not users_exist: + return RedirectResponse(url="/admin/bootstrap", status_code=302) + + candidate = _get_session_candidate_from_request(request) + record = _get_valid_session_record(request) + if record is None: + return RedirectResponse(url="/admin/login", status_code=302) + + user_id = str(record.get("user_id") or "").strip() + if user_id and is_admin_user(user_id): + resp = RedirectResponse(url="/admin/acl", status_code=302) + if candidate.get("source") and candidate.get("source") != "cookie": + _set_admin_session_cookie(resp, str(candidate.get("session_id") or "")) + return resp + return RedirectResponse(url="/admin/login", status_code=302) + + +@app.get("/admin/bootstrap") +async def admin_bootstrap_form(request: Request): + if not AUTH_ENABLED: + raise HTTPException(status_code=404, detail="Auth disabled") + try: + users_exist = has_any_users() + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to inspect user state for bootstrap: {e}") + raise HTTPException(status_code=500, detail="Failed to inspect user state") + if users_exist: + return RedirectResponse(url="/admin/login", status_code=302) + return render_admin_bootstrap(request) + + +@app.post("/admin/bootstrap") +async def admin_bootstrap_submit( + request: Request, + username: str = Form(...), + password: str = Form(...), +): + if not AUTH_ENABLED: + raise HTTPException(status_code=404, detail="Auth disabled") + try: + users_exist = has_any_users() + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to inspect user state for bootstrap submit: {e}") + raise HTTPException(status_code=500, detail="Failed to inspect user state") + if users_exist: + return RedirectResponse(url="/admin/login", status_code=302) + + try: + user = create_user(username, password) + except Exception as e: + return render_admin_error( + request=request, + title="Bootstrap Failed", + message=str(e), + back_href="/admin/bootstrap", + status_code=400, + ) + + try: + session = create_session(user_id=user.get("user_id"), metadata={"client": "admin_ui"}) + except Exception as e: + logger.error(f"[upload_service] Failed to create session after bootstrap: {e}") + raise HTTPException(status_code=500, detail="Failed to create auth session") + + resp = RedirectResponse(url="/admin/acl", status_code=302) + _set_admin_session_cookie(resp, str(session.get("session_id") or "")) + return resp + + +@app.get("/admin/login") +async def admin_login_form(request: Request): + if not AUTH_ENABLED: + raise HTTPException(status_code=404, detail="Auth disabled") + return render_admin_login(request) + + +@app.post("/admin/login") +async def admin_login_submit( + request: Request, + username: str = Form(...), + password: str = Form(...), +): + if not AUTH_ENABLED: + raise HTTPException(status_code=404, detail="Auth disabled") + try: + user = authenticate_user(username, password) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Error authenticating user for admin UI: {e}") + raise HTTPException(status_code=500, detail="Authentication error") + + if not user: + return render_admin_login( + request=request, + error="Invalid credentials", + status_code=401, + ) + + try: + session = create_session(user_id=user.get("id"), metadata={"client": "admin_ui"}) + except Exception as e: + logger.error(f"[upload_service] Failed to create session for admin UI: {e}") + raise HTTPException(status_code=500, detail="Failed to create auth session") + + resp = RedirectResponse(url="/admin/acl", status_code=302) + _set_admin_session_cookie(resp, str(session.get("session_id") or "")) + return resp + + +@app.post("/admin/logout") +async def admin_logout(): + resp = RedirectResponse(url="/admin/login", status_code=302) + resp.delete_cookie(key=ADMIN_SESSION_COOKIE_NAME, path="/") + return resp + + +@app.get("/admin/acl") +async def admin_acl_page(request: Request): + _require_admin_session(request) + try: + users = list_users() + collections = list_collections(include_deleted=False) + grants = list_collection_acl() + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to load admin UI data: {e}") + raise HTTPException(status_code=500, detail="Failed to load admin data") + + resp = render_admin_acl(request, users=users, collections=collections, grants=grants) + candidate = _get_session_candidate_from_request(request) + if candidate.get("source") and candidate.get("source") != "cookie": + _set_admin_session_cookie(resp, str(candidate.get("session_id") or "")) + return resp + + +@app.post("/admin/acl/grant") +async def admin_acl_grant( + request: Request, + user_id: str = Form(...), + collection: str = Form(...), + permission: str = Form("read"), +): + _require_admin_session(request) + try: + grant_collection_access(user_id=user_id, qdrant_collection=collection, permission=permission) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + return render_admin_error(request, title="Grant Failed", message=str(e), back_href="/admin/acl") + return RedirectResponse(url="/admin/acl", status_code=302) + + +@app.post("/admin/users") +async def admin_create_user( + request: Request, + username: str = Form(...), + password: str = Form(...), + role: str = Form("user"), +): + _require_admin_session(request) + try: + create_user(username, password, role=role) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + return render_admin_error(request, title="Create User Failed", message=str(e), back_href="/admin/acl") + return RedirectResponse(url="/admin/acl", status_code=302) + + +@app.post("/admin/acl/revoke") +async def admin_acl_revoke( + request: Request, + user_id: str = Form(...), + collection: str = Form(...), +): + _require_admin_session(request) + try: + revoke_collection_access(user_id=user_id, qdrant_collection=collection) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + return render_admin_error(request, title="Revoke Failed", message=str(e), back_href="/admin/acl") + return RedirectResponse(url="/admin/acl", status_code=302) + + @app.post("/auth/users", response_model=AuthUserCreateResponse) async def auth_create_user(payload: AuthUserCreateRequest, request: Request): try: diff --git a/templates/admin/acl.html b/templates/admin/acl.html new file mode 100644 index 00000000..e3c752f7 --- /dev/null +++ b/templates/admin/acl.html @@ -0,0 +1,110 @@ +{% extends "admin/base.html" %} + +{% block content %} +
+

Users

+ + + + {% if users and users|length > 0 %} + {% for u in users %} + + + + + + {% endfor %} + {% else %} + + {% endif %} +
idusernamerole
{{ u.id }}{{ u.username }}{{ u.role }}
(none)
+ +

Create User

+
+

+

+ +

+ +
+
+ +
+

Collections

+ + + + {% if collections and collections|length > 0 %} + {% for c in collections %} + + + + + {% endfor %} + {% else %} + + {% endif %} +
idqdrant_collection
{{ c.id }}{{ c.qdrant_collection }}
(none)
+
+ +
+

Grants

+ + + + {% if grants and grants|length > 0 %} + {% for g in grants %} + + + + + + + + {% endfor %} + {% else %} + + {% endif %} +
collectionusernameuser_idpermission
{{ g.qdrant_collection }}{{ g.username }}{{ g.user_id }}{{ g.permission }} +
+ + + +
+
(none)
+ +

Grant Collection Access

+
+

+ +

+ +

+ +
+ +

Collection permissions are enforced by MCP servers when CTXCE_MCP_ACL_ENFORCE=1 (and CTXCE_ACL_ALLOW_ALL=0).

+
+{% endblock %} diff --git a/templates/admin/base.html b/templates/admin/base.html new file mode 100644 index 00000000..f3aa77d9 --- /dev/null +++ b/templates/admin/base.html @@ -0,0 +1,39 @@ + + + + + + {{ title }} + + + +
+

{{ title }}

+ +
+ + {% block content %}{% endblock %} + +

CTXCE Admin UI

+ + diff --git a/templates/admin/bootstrap.html b/templates/admin/bootstrap.html new file mode 100644 index 00000000..34b05382 --- /dev/null +++ b/templates/admin/bootstrap.html @@ -0,0 +1,27 @@ +{% extends "admin/base.html" %} + +{% block content %} +
+

Bootstrap Admin User

+ + {% if error %} +

{{ error }}

+ {% endif %} + +
+ +

+ +

+ +
+ +

Only available when no users exist yet.

+
+{% endblock %} diff --git a/templates/admin/error.html b/templates/admin/error.html new file mode 100644 index 00000000..850f623b --- /dev/null +++ b/templates/admin/error.html @@ -0,0 +1,9 @@ +{% extends "admin/base.html" %} + +{% block content %} +
+

{{ title }}

+

{{ message }}

+

Back

+
+{% endblock %} diff --git a/templates/admin/login.html b/templates/admin/login.html new file mode 100644 index 00000000..1b94be36 --- /dev/null +++ b/templates/admin/login.html @@ -0,0 +1,25 @@ +{% extends "admin/base.html" %} + +{% block content %} +
+

Login

+ + {% if error %} +

{{ error }}

+ {% endif %} + +
+ +

+ +

+ +
+
+{% endblock %} From 21683bb26fe939617edbfd24295a8a5c0288c7aa Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 13:38:42 +0000 Subject: [PATCH 42/55] fix: init container chmod and user --- docker-compose.dev-remote.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docker-compose.dev-remote.yml b/docker-compose.dev-remote.yml index ef80cf3b..2154ca1b 100644 --- a/docker-compose.dev-remote.yml +++ b/docker-compose.dev-remote.yml @@ -324,7 +324,7 @@ services: context: . dockerfile: Dockerfile.indexer container_name: init-payload-dev-remote - user: "1000:1000" + user: "0:0" depends_on: - qdrant env_file: @@ -345,7 +345,7 @@ services: command: [ "sh", "-c", - "mkdir -p /tmp/logs && echo 'Starting initialization sequence...' && /app/scripts/wait-for-qdrant.sh && PYTHONPATH=/app python /app/scripts/create_indexes.py && echo 'Collections and metadata created' && python /app/scripts/warm_all_collections.py && echo 'Search caches warmed for all collections' && python /app/scripts/health_check.py && echo 'Initialization completed successfully!'" + "mkdir -p /tmp/logs /work/.codebase && (chgrp -R 1000 /work/.codebase 2>/dev/null || true) && (chmod -R g+rwX /work/.codebase 2>/dev/null || true) && (find /work/.codebase -type d -exec chmod g+s {} + 2>/dev/null || true) && echo 'Starting initialization sequence...' && /app/scripts/wait-for-qdrant.sh && PYTHONPATH=/app python /app/scripts/create_indexes.py && echo 'Collections and metadata created' && python /app/scripts/warm_all_collections.py && echo 'Search caches warmed for all collections' && python /app/scripts/health_check.py && echo 'Initialization completed successfully!'" ] restart: "no" # Run once on startup networks: @@ -411,6 +411,11 @@ services: - workspace_pvc:/work:rw - codebase_pvc:/work/.codebase:rw - upload_temp:/tmp/uploads + command: [ + "sh", + "-c", + "mkdir -p /work/.codebase && (chgrp -R 1000 /work/.codebase 2>/dev/null || true) && (chmod -R g+rwX /work/.codebase 2>/dev/null || true) && (find /work/.codebase -type d -exec chmod g+s {} + 2>/dev/null || true) && exec python scripts/upload_service.py" + ] healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8002/health"] interval: 30s From 9c041e7c57d36ecd1f6c59b2f57ca255086ed0c9 Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 13:38:42 +0000 Subject: [PATCH 43/55] feat: enable auth env vars for init container --- docker-compose.dev-remote.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker-compose.dev-remote.yml b/docker-compose.dev-remote.yml index 2154ca1b..7443c5b1 100644 --- a/docker-compose.dev-remote.yml +++ b/docker-compose.dev-remote.yml @@ -332,6 +332,13 @@ services: environment: - QDRANT_URL=${QDRANT_URL} - COLLECTION_NAME=${COLLECTION_NAME} + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} - HF_HOME=/work/.cache/huggingface - TRANSFORMERS_CACHE=/work/.cache/huggingface - HUGGINGFACE_HUB_CACHE=/work/.cache/huggingface From eef74527aa5204db2dabe804ddc79f98647a7344 Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 13:38:43 +0000 Subject: [PATCH 44/55] feat: set HNSW index params when creating collections --- scripts/mcp_memory_server.py | 6 +++++- scripts/memory_restore.py | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/mcp_memory_server.py b/scripts/mcp_memory_server.py index e976b399..14ad3d0c 100644 --- a/scripts/mcp_memory_server.py +++ b/scripts/mcp_memory_server.py @@ -281,7 +281,11 @@ def _ensure_collection(name: str): except Exception: pass - client.create_collection(collection_name=name, vectors_config=vectors_cfg) + client.create_collection( + collection_name=name, + vectors_config=vectors_cfg, + hnsw_config=models.HnswConfigDiff(m=16, ef_construct=256), + ) vector_names = list(vectors_cfg.keys()) print(f"[MEMORY_SERVER] Created collection '{name}' with vectors: {vector_names}") return True diff --git a/scripts/memory_restore.py b/scripts/memory_restore.py index c2f4c01e..d8c698f9 100644 --- a/scripts/memory_restore.py +++ b/scripts/memory_restore.py @@ -84,9 +84,10 @@ def ensure_collection_exists( size=vector_dimension, distance=Distance.COSINE ) - } + }, + hnsw_config=HnswConfigDiff(m=16, ef_construct=256), ) - print(f"✅ Created collection '{collection_name}' with {vector_dimension}-dim vectors") + print(f"Created collection '{collection_name}' with {vector_dimension}-dim vectors") except Exception as e: raise RuntimeError(f"Failed to create collection '{collection_name}': {e}") From 6f4296b32bfca2f15ee99c717996fbdf3b5eca15 Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 13:38:43 +0000 Subject: [PATCH 45/55] feat: mcp memory server fastembed cache --- scripts/mcp_memory_server.py | 44 ++++++------------------------------ 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/scripts/mcp_memory_server.py b/scripts/mcp_memory_server.py index 14ad3d0c..b557457e 100644 --- a/scripts/mcp_memory_server.py +++ b/scripts/mcp_memory_server.py @@ -32,27 +32,8 @@ EMBEDDING_MODEL = os.environ.get("EMBEDDING_MODEL", "BAAI/bge-base-en-v1.5") # Minimal embedding via fastembed (CPU) -from fastembed import TextEmbedding # Single-process embedding model cache (avoid re-initializing fastembed on each call) -_EMBED_MODEL = None -_EMBED_LOCK = threading.Lock() - -def _get_embedding_model(): - global _EMBED_MODEL - m = _EMBED_MODEL - if m is None: - with _EMBED_LOCK: - m = _EMBED_MODEL - if m is None: - m = TextEmbedding(model_name=EMBEDDING_MODEL) - # Best-effort warmup to load weights once - try: - _ = list(m.embed(["memory", "search"])) - except Exception: - pass - _EMBED_MODEL = m - return m # Ensure repo roots are importable so 'scripts' resolves inside container import sys as _sys @@ -101,6 +82,10 @@ def _get_embedding_model(): m = _EMBED_MODEL_CACHE.get(EMBEDDING_MODEL) if m is None: m = TextEmbedding(model_name=EMBEDDING_MODEL) + try: + _ = next(m.embed(["warmup"])) + except Exception: + pass _EMBED_MODEL_CACHE[EMBEDDING_MODEL] = m return m @@ -157,26 +142,8 @@ def _inner(fn): _SESSION_CTX_LOCK = threading.Lock() SESSION_DEFAULTS_BY_SESSION: "WeakKeyDictionary[Any, Dict[str, Any]]" = WeakKeyDictionary() -try: - from scripts.auth_backend import AUTH_ENABLED as AUTH_ENABLED_AUTH, validate_session as _auth_validate_session -except Exception: - AUTH_ENABLED_AUTH = False - - def _auth_validate_session(session_id: str): # type: ignore[no-redef] - return None -def _require_auth_session(session: Optional[str]) -> Optional[Dict[str, Any]]: - if not AUTH_ENABLED_AUTH: - return None - sid = (session or "").strip() - if not sid: - raise Exception("Missing session for authorized operation") - info = _auth_validate_session(sid) - if not info: - raise Exception("Invalid or expired session") - return info - def _start_readyz_server(): try: @@ -243,6 +210,9 @@ def _ensure_collection(name: str): # Choose dense dimension based on config: probe (default) vs env-configured if MEMORY_PROBE_EMBED_DIM: try: + # Probe dimension without populating the shared model cache. + # This preserves the "cache loads on first tool call" behavior and + # keeps MEMORY_COLD_SKIP_DENSE semantics unchanged. from fastembed import TextEmbedding _model_probe = TextEmbedding(model_name=EMBEDDING_MODEL) _dense_vec = next(_model_probe.embed(["probe"])) From ff005034142d5e37674f2af08b2db30692fa2d27 Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 13:38:43 +0000 Subject: [PATCH 46/55] feat: Enforce collection access on memory find and store endpoints and upload service --- scripts/mcp_memory_server.py | 7 ++++--- scripts/upload_service.py | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/scripts/mcp_memory_server.py b/scripts/mcp_memory_server.py index b557457e..88ad9157 100644 --- a/scripts/mcp_memory_server.py +++ b/scripts/mcp_memory_server.py @@ -353,8 +353,9 @@ def store( First call may be slower because the embedding model loads lazily. """ - _require_auth_session(session) + sess = _require_auth_session(session) coll = _resolve_collection(collection, session=session, ctx=ctx, extra_kwargs=kwargs) + _require_collection_access((sess or {}).get("user_id"), coll, "write") _ensure_once(coll) model = _get_embedding_model() dense = next(model.embed([str(information)])).tolist() @@ -388,9 +389,9 @@ def find( Cold-start option: set MEMORY_COLD_SKIP_DENSE=1 to skip dense embedding until the model is cached (useful on slow storage). """ - # _require_auth_session(session) # TODO: - + sess = _require_auth_session(session) coll = _resolve_collection(collection, session=session, ctx=ctx, extra_kwargs=kwargs) + _require_collection_access((sess or {}).get("user_id") if sess else None, coll, "read") _ensure_once(coll) use_dense = True diff --git a/scripts/upload_service.py b/scripts/upload_service.py index d5aee9b5..ce748237 100644 --- a/scripts/upload_service.py +++ b/scripts/upload_service.py @@ -30,9 +30,22 @@ create_session_for_token, create_user, has_any_users, + has_collection_access, validate_session, AUTH_ENABLED, AUTH_SESSION_TTL_SECONDS, + is_admin_user, + list_users, + list_collections, + list_collection_acl, + grant_collection_access, + revoke_collection_access, +) +from scripts.admin_ui import ( + render_admin_acl, + render_admin_bootstrap, + render_admin_error, + render_admin_login, ) # Import existing workspace state and indexing functions @@ -930,6 +943,8 @@ async def upload_delta_bundle( start_time = datetime.now() client_host = request.client.host if hasattr(request, 'client') and request.client else 'unknown' + record: Optional[Dict[str, Any]] = None + try: logger.info(f"[upload_service] Begin processing upload for workspace={workspace_path} from {client_host}") @@ -1013,6 +1028,28 @@ async def upload_delta_bundle( else: collection_name = DEFAULT_COLLECTION + # Enforce collection write access for uploads when auth is enabled. + # Semantics: "write" is sufficient for uploading/indexing content. + if AUTH_ENABLED and CTXCE_MCP_ACL_ENFORCE: + uid = str((record or {}).get("user_id") or "").strip() + if not uid: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired session", + ) + try: + allowed = has_collection_access(uid, str(collection_name), "write") + except AuthDisabledError: + allowed = True + except Exception as e: + logger.error(f"[upload_service] Failed to check collection access for upload: {e}") + raise HTTPException(status_code=500, detail="Failed to check collection access") + if not allowed: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Forbidden: write access to collection '{collection_name}' denied", + ) + # Persist origin metadata for remote lookups (including client source_path) # Use slugged repo name (repo+16) for state so it matches ingest/watch_index usage try: From d52ccb340398e89818117faf1b1c97c971ad4ba6 Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 13:38:43 +0000 Subject: [PATCH 47/55] fix: mcp_indexer_server require session for index commands --- scripts/mcp_indexer_server.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/mcp_indexer_server.py b/scripts/mcp_indexer_server.py index ce47d0c9..24449757 100644 --- a/scripts/mcp_indexer_server.py +++ b/scripts/mcp_indexer_server.py @@ -958,7 +958,7 @@ def _detect_current_repo() -> str | None: @mcp.tool() async def qdrant_index_root( - recreate: Optional[bool] = None, collection: Optional[str] = None + recreate: Optional[bool] = None, collection: Optional[str] = None, session: Optional[str] = None ) -> Dict[str, Any]: """Initialize or refresh the vector index for the workspace root (/work). @@ -976,6 +976,8 @@ async def qdrant_index_root( - Omit fields instead of sending null values. - Safe to call repeatedly; unchanged files are skipped by the indexer. """ + sess = _require_auth_session(session) + # Leniency: if clients embed JSON in 'collection' (and include 'recreate'), parse it try: if _looks_jsonish_string(collection): @@ -1573,6 +1575,7 @@ async def qdrant_index( subdir: Optional[str] = None, recreate: Optional[bool] = None, collection: Optional[str] = None, + session: Optional[str] = None, ) -> Dict[str, Any]: """Index the workspace (/work) or a specific subdirectory. @@ -1589,6 +1592,8 @@ async def qdrant_index( - Paths are sandboxed to /work; attempts to escape will be rejected. - Omit fields rather than sending null values. """ + sess = _require_auth_session(session) + # Leniency: parse JSON-ish payloads mistakenly sent in 'collection' or 'subdir' try: if _looks_jsonish_string(collection): From 016f24842a7e108eefd9a988d0947398e6eb8d3b Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 13:38:43 +0000 Subject: [PATCH 48/55] fix: memory_restore output --- scripts/memory_restore.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/memory_restore.py b/scripts/memory_restore.py index d8c698f9..c85e925e 100644 --- a/scripts/memory_restore.py +++ b/scripts/memory_restore.py @@ -27,7 +27,7 @@ try: from qdrant_client import QdrantClient - from qdrant_client.models import VectorParams, Distance + from qdrant_client.models import VectorParams, Distance, HnswConfigDiff from fastembed import TextEmbedding except ImportError as e: print(f"ERROR: Missing required dependency: {e}") @@ -271,7 +271,7 @@ def restore_memories( error_count += len(batch_points) # Final statistics - print(f"\n✅ Memory restore completed!") + print(f"\n Memory restore completed!") print(f" Total memories in backup: {len(memories)}") print(f" Successfully restored: {restored_count}") print(f" Skipped (already exists): {skipped_count}") @@ -395,13 +395,13 @@ def main(): ) if result["success"]: - print(f"\n🎉 Memory restoration completed successfully!") + print(f"\n Memory restoration completed successfully!") else: - print(f"\n❌ Memory restoration failed!") + print(f"\n Memory restoration failed!") sys.exit(1) except Exception as e: - print(f"\n❌ Error during restoration: {e}") + print(f"\n Error during restoration: {e}") sys.exit(1) From 71ee135926f2e0dd8d32a413763ad1e65a604f42 Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 13:38:43 +0000 Subject: [PATCH 49/55] fix: mcp_indexer_server enforce auth for search commands --- scripts/mcp_indexer_server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/mcp_indexer_server.py b/scripts/mcp_indexer_server.py index 24449757..9df8fcfd 100644 --- a/scripts/mcp_indexer_server.py +++ b/scripts/mcp_indexer_server.py @@ -1831,8 +1831,7 @@ async def repo_search( - path_glob=["scripts/**","**/*.py"], language="python" - symbol="context_answer", under="scripts" """ - # Enforce auth when enabled (no-op when CTXCE_AUTH_ENABLED is false) - _require_auth_session(session) + sess = _require_auth_session(session) # Handle queries alias (explicit parameter) if queries is not None and (query is None or (isinstance(query, str) and str(query).strip() == "")): From ec60fe164a5b1459cd02c4cdca7c9a2884077e62 Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 20:09:42 +0000 Subject: [PATCH 50/55] k8: update configmap, env/sample update, fix memory mcp crash --- .env | 9 +++++++++ .env.example | 5 +++-- deploy/kubernetes/configmap.yaml | 27 +++++++++++++++++++++++++-- scripts/mcp_memory_server.py | 25 +++++++++++-------------- scripts/sync_env_to_k8s.py | 0 5 files changed, 48 insertions(+), 18 deletions(-) mode change 100644 => 100755 scripts/sync_env_to_k8s.py diff --git a/.env b/.env index 9695c8a8..13085612 100644 --- a/.env +++ b/.env @@ -128,6 +128,7 @@ REFRAG_RUNTIME=llamacpp REFRAG_ENCODER_MODEL=BAAI/bge-base-en-v1.5 REFRAG_PHI_PATH=/work/models/refrag_phi_768_to_dmodel.bin REFRAG_SENSE=heuristic +REFRAG_PSEUDO_DESCRIBE=1 GLM_API_KEY= # Llama.cpp sidecar (optional) # Use docker network hostname from containers; localhost remains ok for host-side runs if LLAMACPP_URL not exported @@ -135,6 +136,7 @@ LLAMACPP_URL=http://host.docker.internal:8081 LLAMACPP_TIMEOUT_SEC=300 DECODER_MAX_TOKENS=4000 REFRAG_DECODER_MODE=prompt # prompt|soft +REFRAG_COMMIT_DESCRIBE=1 REFRAG_SOFT_SCALE=1.0 LLAMACPP_USE_GPU=1 @@ -149,6 +151,8 @@ MAX_MICRO_CHUNKS_PER_FILE=500 QDRANT_TIMEOUT=20 MEMORY_AUTODETECT=1 MEMORY_COLLECTION_TTL_SECS=300 +SMART_SYMBOL_REINDEXING=1 +MAX_CHANGED_SYMBOLS_RATIO=0.6 # Watcher-safe defaults (recommended) @@ -196,4 +200,9 @@ INFO_REQUEST_LIMIT=10 INFO_REQUEST_CONTEXT_LINES=5 # INFO_REQUEST_EXPLAIN_DEFAULT=0 # INFO_REQUEST_RELATIONSHIPS=0 +GLM_API_BASE=https://api.z.ai/api/coding/paas/v4/ +GLM_MODEL=glm-4.6 COMMIT_VECTOR_SEARCH=0 +STRICT_MEMORY_RESTORE=1 +CTXCE_AUTH_ENABLED=0 +CTXCE_AUTH_ADMIN_TOKEN=change-me-admin-token diff --git a/.env.example b/.env.example index ced87907..c759cff8 100644 --- a/.env.example +++ b/.env.example @@ -152,7 +152,7 @@ REFRAG_PHI_PATH=/work/models/refrag_phi_768_to_dmodel.json REFRAG_SENSE=heuristic # Enable index-time pseudo descriptions for micro-chunks (requires REFRAG_DECODER) -# REFRAG_PSEUDO_DESCRIBE=1 +REFRAG_PSEUDO_DESCRIBE=1 # Llama.cpp sidecar (optional) # Docker CPU-only (stable): http://llamacpp:8080 @@ -190,6 +190,7 @@ MEMORY_COLLECTION_TTL_SECS=300 # Smarter re-indexing for symbol cache, reuse embeddings and reduce decoder/pseudo tags to re-index SMART_SYMBOL_REINDEXING=1 +MAX_CHANGED_SYMBOLS_RATIO=0.6 # Watcher-safe defaults (recommended) # Applied to watcher via compose; uncomment to apply globally. @@ -226,7 +227,7 @@ SMART_SYMBOL_REINDEXING=1 REFRAG_COMMIT_DESCRIBE=1 COMMIT_VECTOR_SEARCH=0 -STRICT_MEMORY_RESTORE=0 +STRICT_MEMORY_RESTORE=1 # info_request() tool settings (simplified codebase retrieval) # Default result limit for info_request queries diff --git a/deploy/kubernetes/configmap.yaml b/deploy/kubernetes/configmap.yaml index 34fe12c5..ec5bbc67 100644 --- a/deploy/kubernetes/configmap.yaml +++ b/deploy/kubernetes/configmap.yaml @@ -8,8 +8,12 @@ metadata: component: configuration data: COLLECTION_NAME: codebase + COMMIT_VECTOR_SEARCH: '0' + CTXCE_AUTH_ADMIN_TOKEN: change-me-admin-token + CTXCE_AUTH_ENABLED: '0' CTX_SNIPPET_CHARS: '400' CTX_SUMMARY_CHARS: '0' + CURRENT_REPO: '' DECODER_MAX_TOKENS: '4000' EMBEDDING_MODEL: BAAI/bge-base-en-v1.5 EMBEDDING_PROVIDER: fastembed @@ -40,6 +44,11 @@ data: INDEX_UPSERT_BACKOFF: '0.5' INDEX_UPSERT_BATCH: '128' INDEX_UPSERT_RETRIES: '5' + INDEX_USE_ENHANCED_AST: '1' + INFO_REQUEST_CONTEXT_LINES: '5' + INFO_REQUEST_EXPLAIN_DEFAULT: '0' + INFO_REQUEST_LIMIT: '10' + INFO_REQUEST_RELATIONSHIPS: '0' LLAMACPP_EXTRA_ARGS: '' LLAMACPP_GPU_LAYERS: '32' LLAMACPP_GPU_SPLIT: '' @@ -70,10 +79,22 @@ data: MINI_VEC_SEED: '1337' MULTI_REPO_MODE: '1' OLLAMA_HOST: http://host.docker.internal:11434 + POST_RERANK_SYMBOL_BOOST: '1.0' PRF_ENABLED: '1' QDRANT_API_KEY: '' + QDRANT_EF_SEARCH: '128' QDRANT_TIMEOUT: '20' QDRANT_URL: http://qdrant:6333 + QUERY_OPTIMIZER_ADAPTIVE: '1' + QUERY_OPTIMIZER_COLLECTION_SIZE: '10000' + QUERY_OPTIMIZER_COMPLEX_FACTOR: '2.0' + QUERY_OPTIMIZER_COMPLEX_THRESHOLD: '0.7' + QUERY_OPTIMIZER_DENSE_THRESHOLD: '0.2' + QUERY_OPTIMIZER_MAX_EF: '512' + QUERY_OPTIMIZER_MIN_EF: '64' + QUERY_OPTIMIZER_SEMANTIC_FACTOR: '1.0' + QUERY_OPTIMIZER_SIMPLE_FACTOR: '0.5' + QUERY_OPTIMIZER_SIMPLE_THRESHOLD: '0.3' REFRAG_CANDIDATES: '200' REFRAG_COMMIT_DESCRIBE: '1' REFRAG_DECODER: '1' @@ -87,12 +108,14 @@ data: REFRAG_SENSE: heuristic REFRAG_SOFT_SCALE: '1.0' REMOTE_UPLOAD_GIT_MAX_COMMITS: '500' + REPO_AUTO_FILTER: '1' RERANKER_ENABLED: '1' - RERANKER_ONNX_PATH: /work/models/model_qint8_avx512_vnni.onnx + RERANKER_ONNX_PATH: /app/models/reranker.onnx RERANKER_RETURN_M: '20' RERANKER_TIMEOUT_MS: '3000' - RERANKER_TOKENIZER_PATH: /work/models/tokenizer.json + RERANKER_TOKENIZER_PATH: /app/models/tokenizer.json RERANKER_TOPN: '100' + RERANK_BLEND_WEIGHT: '0.6' RERANK_EXPAND: '1' RERANK_IN_PROCESS: '1' RERANK_TIMEOUT_FLOOR_MS: '1000' diff --git a/scripts/mcp_memory_server.py b/scripts/mcp_memory_server.py index 88ad9157..9fbcd404 100644 --- a/scripts/mcp_memory_server.py +++ b/scripts/mcp_memory_server.py @@ -4,6 +4,17 @@ import threading from weakref import WeakKeyDictionary +# Ensure repo roots are importable so 'scripts' resolves inside container +import sys as _sys +_roots_env = os.environ.get("WORK_ROOTS", "") +_roots = [p.strip() for p in _roots_env.split(",") if p.strip()] or ["/work", "/app"] +try: + for _root in _roots: + if _root and _root not in _sys.path: + _sys.path.insert(0, _root) +except Exception: + pass + # FastMCP server and request Context (ctx) for per-connection state try: @@ -35,20 +46,6 @@ # Single-process embedding model cache (avoid re-initializing fastembed on each call) -# Ensure repo roots are importable so 'scripts' resolves inside container -import sys as _sys -_roots_env = os.environ.get("WORK_ROOTS", "") -_roots = [p.strip() for p in _roots_env.split(",") if p.strip()] or ["/work", "/app"] -try: - for _root in _roots: - if _root and _root not in _sys.path: - _sys.path.insert(0, _root) -except Exception: - pass - -# Map model to named vector used in indexer - - # Use shared utils for consistent vector naming and lexical hashing from scripts.utils import sanitize_vector_name as _sanitize_vector_name from scripts.utils import lex_hash_vector_text as _lex_hash_vector_text diff --git a/scripts/sync_env_to_k8s.py b/scripts/sync_env_to_k8s.py old mode 100644 new mode 100755 From 4a974988bce4a954b447e4947051f2714773d63d Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 20:46:31 +0000 Subject: [PATCH 51/55] fix(auth): restore session DB init + fail closed if auth backend import breaks - Fix /auth/login/password 500 by calling _ensure_db() in auth_backend.create_session() (was calling undefined _ensure_auth_db()). - Harden scripts/mcp_auth.py fallback behavior: when CTXCE_AUTH_ENABLED=1 and auth_backend import fails, raise ValidationError instead of silently allowing all session/ACL checks. --- scripts/auth_backend.py | 2 +- scripts/mcp_auth.py | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/auth_backend.py b/scripts/auth_backend.py index f9e3fe6a..d298f6d2 100644 --- a/scripts/auth_backend.py +++ b/scripts/auth_backend.py @@ -258,7 +258,7 @@ def create_session( ) -> Dict[str, Any]: if not AUTH_ENABLED: raise AuthDisabledError("Auth not enabled") - _ensure_auth_db() + _ensure_db() path = _get_auth_db_path() now_ts = int(datetime.now().timestamp()) ttl_val = int(ttl_seconds or 0) diff --git a/scripts/mcp_auth.py b/scripts/mcp_auth.py index 06ccc0cb..2b13c791 100644 --- a/scripts/mcp_auth.py +++ b/scripts/mcp_auth.py @@ -16,16 +16,29 @@ class ValidationError(Exception): validate_session as _auth_validate_session, has_collection_access as _has_collection_access, ) -except Exception: - AUTH_ENABLED_AUTH = False - ACL_ALLOW_ALL_AUTH = True +except Exception as _auth_backend_import_exc: + _AUTH_BACKEND_IMPORT_ERROR = repr(_auth_backend_import_exc) + AUTH_ENABLED_AUTH = ( + str(os.environ.get("CTXCE_AUTH_ENABLED", "0")).strip().lower() in {"1", "true", "yes", "on"} + ) + ACL_ALLOW_ALL_AUTH = ( + str(os.environ.get("CTXCE_ACL_ALLOW_ALL", "0")).strip().lower() in {"1", "true", "yes", "on"} + ) def _auth_validate_session(session_id: str): # type: ignore[no-redef] + if AUTH_ENABLED_AUTH: + raise ValidationError( + f"Auth backend unavailable (import failed): {_AUTH_BACKEND_IMPORT_ERROR}" + ) return None def _has_collection_access( user_id: str, qdrant_collection: str, permission: str = "read" ) -> bool: # type: ignore[no-redef] + if AUTH_ENABLED_AUTH: + raise ValidationError( + f"Auth backend unavailable (import failed): {_AUTH_BACKEND_IMPORT_ERROR}" + ) return True From 9a27650c1698513fdeb9f226efba8d3f60b2dfa8 Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 21:35:09 +0000 Subject: [PATCH 52/55] Fix repo_search rerank to respect explicit Qdrant collection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ensure in-process dense rerank (rerank_in_process) queries the request’s collection instead of falling back to COLLECTION_NAME. Thread collection through rerank paths: pass collection into rerank_in_process from mcp_indexer_server.repo_search pass --collection to the rerank subprocess for consistency Add regression tests to prevent cross-collection result leakage when rerank_enabled=true. --- scripts/mcp_indexer_server.py | 3 ++ scripts/rerank_local.py | 30 ++++++++++++----- tests/test_reranker_verification.py | 52 ++++++++++++++++++++++++++++- 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/scripts/mcp_indexer_server.py b/scripts/mcp_indexer_server.py index 9df8fcfd..c3da499f 100644 --- a/scripts/mcp_indexer_server.py +++ b/scripts/mcp_indexer_server.py @@ -2515,6 +2515,7 @@ def _doc_for(obj: dict) -> str: language=language or None, under=under or None, model=model, + collection=collection, ) if items: results = items @@ -2535,6 +2536,8 @@ def _doc_for(obj: dict) -> str: "--limit", str(int(rerank_return_m)), ] + if collection: + rcmd += ["--collection", str(collection)] if language: rcmd += ["--language", language] if under: diff --git a/scripts/rerank_local.py b/scripts/rerank_local.py index 035d9467..4a0f8790 100644 --- a/scripts/rerank_local.py +++ b/scripts/rerank_local.py @@ -14,7 +14,6 @@ ort = None Tokenizer = None -COLLECTION = os.environ.get("COLLECTION_NAME", "codebase") MODEL_NAME = os.environ.get("EMBEDDING_MODEL", "BAAI/bge-base-en-v1.5") QDRANT_URL = os.environ.get("QDRANT_URL", "http://localhost:6333") API_KEY = os.environ.get("QDRANT_API_KEY") @@ -175,11 +174,12 @@ def dense_results( query: str, flt, topk: int, + collection_name: str, ) -> List[Any]: vec = next(model.embed([query])).tolist() try: qp = client.query_points( - collection_name=COLLECTION, + collection_name=collection_name, query=vec, using=vec_name, query_filter=flt, @@ -190,7 +190,7 @@ def dense_results( return getattr(qp, "points", qp) except Exception: res = client.search( - collection_name=COLLECTION, + collection_name=collection_name, query_vector={"name": vec_name, "vector": vec}, limit=topk, with_payload=True, @@ -275,11 +275,17 @@ def rerank_in_process( language: str | None = None, under: str | None = None, model: TextEmbedding | None = None, + collection: str | None = None, ) -> List[Dict[str, Any]]: + eff_collection = ( + str(collection).strip() + if isinstance(collection, str) and collection.strip() + else (os.environ.get("COLLECTION_NAME") or "codebase") + ) client = QdrantClient(url=QDRANT_URL, api_key=API_KEY or None) _model = model or TextEmbedding(model_name=MODEL_NAME) dim = len(next(_model.embed(["dimension probe"]))) - vec_name = _select_dense_vector_name(client, COLLECTION, _model, dim) + vec_name = _select_dense_vector_name(client, eff_collection, _model, dim) must = [] if language: @@ -297,9 +303,9 @@ def rerank_in_process( ) flt = models.Filter(must=must) if must else None - pts = dense_results(client, _model, vec_name, query, flt, topk) + pts = dense_results(client, _model, vec_name, query, flt, topk, eff_collection) if not pts and flt is not None: - pts = dense_results(client, _model, vec_name, query, None, topk) + pts = dense_results(client, _model, vec_name, query, None, topk, eff_collection) if not pts: return [] @@ -330,13 +336,19 @@ def main(): ap.add_argument("--limit", type=int, default=12) ap.add_argument("--language", type=str, default=None) ap.add_argument("--under", type=str, default=None) + ap.add_argument("--collection", type=str, default=None) args = ap.parse_args() client = QdrantClient(url=QDRANT_URL, api_key=API_KEY or None) model = TextEmbedding(model_name=MODEL_NAME) dim = len(next(model.embed(["dimension probe"]))) - vec_name = _select_dense_vector_name(client, COLLECTION, model, dim) + eff_collection = ( + str(args.collection).strip() + if isinstance(args.collection, str) and args.collection.strip() + else (os.environ.get("COLLECTION_NAME") or "codebase") + ) + vec_name = _select_dense_vector_name(client, eff_collection, model, dim) must = [] if args.language: @@ -354,10 +366,10 @@ def main(): ) flt = models.Filter(must=must) if must else None - pts = dense_results(client, model, vec_name, args.query, flt, args.topk) + pts = dense_results(client, model, vec_name, args.query, flt, args.topk, eff_collection) # Fallback: if filtered search yields nothing, retry without filters to avoid empty rerank if not pts and flt is not None: - pts = dense_results(client, model, vec_name, args.query, None, args.topk) + pts = dense_results(client, model, vec_name, args.query, None, args.topk, eff_collection) if not pts: return pairs = prepare_pairs(args.query, pts) diff --git a/tests/test_reranker_verification.py b/tests/test_reranker_verification.py index 2642c65f..e7a05245 100644 --- a/tests/test_reranker_verification.py +++ b/tests/test_reranker_verification.py @@ -108,6 +108,45 @@ def fake_rerank_local(pairs): assert [r["path"] for r in rr["results"]] == ["/work/b.py", "/work/a.py"] +@pytest.mark.service +@pytest.mark.anyio +async def test_rerank_inproc_dense_respects_collection_argument(monkeypatch): + # Drive the in-process dense rerank fallback path by returning no hybrid candidates. + monkeypatch.setenv("HYBRID_IN_PROCESS", "1") + monkeypatch.setenv("RERANK_IN_PROCESS", "1") + + def fake_run_hybrid_search(**kwargs): + return [] + + monkeypatch.setitem(sys.modules, "scripts.hybrid_search", _make_hybrid_stub(fake_run_hybrid_search)) + monkeypatch.delitem(sys.modules, "scripts.mcp_indexer_server", raising=False) + server = importlib.import_module("scripts.mcp_indexer_server") + monkeypatch.setattr(server, "_get_embedding_model", _fake_embedding_model) + + captured = {} + + def fake_rerank_in_process(**kwargs): + captured.update(kwargs) + return [] + + monkeypatch.setattr( + importlib.import_module("scripts.rerank_local"), + "rerank_in_process", + fake_rerank_in_process, + ) + + await server.repo_search( + query="q", + limit=2, + per_path=2, + rerank_enabled=True, + compact=True, + collection="other-collection", + ) + + assert captured.get("collection") == "other-collection" + + @pytest.mark.service @pytest.mark.anyio async def test_rerank_subprocess_timeout_fallback(monkeypatch): @@ -122,6 +161,10 @@ def fake_run_hybrid_search(**kwargs): ] async def fake_run_async(cmd, env=None, timeout=None): + # Ensure explicit collection is forwarded to subprocess reranker + assert "--collection" in cmd + idx = cmd.index("--collection") + assert cmd[idx + 1] == "test-coll" # Simulate subprocess reranker timing out return {"ok": False, "code": -1, "stdout": "", "stderr": f"Command timed out after {timeout}s"} @@ -137,7 +180,14 @@ async def fake_run_async(cmd, env=None, timeout=None): monkeypatch.setattr(server, "_get_embedding_model", _fake_embedding_model) monkeypatch.setattr(server, "_run_async", fake_run_async) - rr = await server.repo_search(query="q", limit=2, per_path=2, rerank_enabled=True, compact=True) + rr = await server.repo_search( + query="q", + limit=2, + per_path=2, + rerank_enabled=True, + compact=True, + collection="test-coll", + ) # Fallback should keep original order from hybrid; timeout counter incremented assert rr.get("used_rerank") is False assert rr.get("rerank_counters", {}).get("timeout", 0) >= 1 From 896e4a5bfd2cc899464546aa18be1390df80ddfc Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 22:33:33 +0000 Subject: [PATCH 53/55] fix: enhance security with upload_service and stability - Add optional admin_ui imports with graceful fallback - Implement path traversal protection in delta bundle processing - Add missing contextlib import Prevents directory traversal attacks and improves service resilience when admin UI components are unavailable. --- scripts/subprocess_manager.py | 1 + scripts/upload_service.py | 61 +++++++++++---- tests/test_upload_service_path_traversal.py | 84 +++++++++++++++++++++ 3 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 tests/test_upload_service_path_traversal.py diff --git a/scripts/subprocess_manager.py b/scripts/subprocess_manager.py index 0bf89f3b..aaf2904f 100644 --- a/scripts/subprocess_manager.py +++ b/scripts/subprocess_manager.py @@ -9,6 +9,7 @@ from typing import Optional, Dict, Any, List, Union import logging import os +import contextlib logger = logging.getLogger(__name__) diff --git a/scripts/upload_service.py b/scripts/upload_service.py index ce748237..36255f07 100644 --- a/scripts/upload_service.py +++ b/scripts/upload_service.py @@ -41,12 +41,22 @@ grant_collection_access, revoke_collection_access, ) -from scripts.admin_ui import ( - render_admin_acl, - render_admin_bootstrap, - render_admin_error, - render_admin_login, -) +try: + from scripts.admin_ui import ( + render_admin_acl, + render_admin_bootstrap, + render_admin_error, + render_admin_login, + ) +except Exception: + + def _admin_ui_unavailable(*args, **kwargs): + raise HTTPException(status_code=500, detail="Admin UI unavailable") + + render_admin_acl = _admin_ui_unavailable + render_admin_bootstrap = _admin_ui_unavailable + render_admin_error = _admin_ui_unavailable + render_admin_login = _admin_ui_unavailable # Import existing workspace state and indexing functions try: @@ -355,6 +365,24 @@ def process_delta_bundle(workspace_path: str, bundle_path: Path, manifest: Dict[ workspace.mkdir(parents=True, exist_ok=True) slug_repo_name = f"{repo_name}-{workspace_key}" + workspace_root = workspace.resolve() + + def _safe_join(base: Path, rel: str) -> Path: + rp = Path(str(rel)) + if str(rp) in {".", ""}: + raise ValueError("Invalid operation path") + if rp.is_absolute(): + raise ValueError(f"Absolute paths are not allowed: {rel}") + base_resolved = base.resolve() + candidate = (base_resolved / rp).resolve() + try: + ok = candidate.is_relative_to(base_resolved) + except Exception: + ok = os.path.commonpath([str(base_resolved), str(candidate)]) == str(base_resolved) + if not ok: + raise ValueError(f"Path escapes workspace: {rel}") + return candidate + with tarfile.open(bundle_path, "r:gz") as tar: # Extract operations metadata ops_member = None @@ -419,7 +447,16 @@ def process_delta_bundle(workspace_path: str, bundle_path: Path, manifest: Dict[ logger.error(msg) raise ValueError(msg) - target_path = workspace / rel_path + target_path = _safe_join(workspace_root, rel_path) + + safe_source_path = None + source_rel_path = None + if op_type == "moved": + source_rel_path = operation.get("source_path") or operation.get( + "source_relative_path" + ) + if source_rel_path: + safe_source_path = _safe_join(workspace_root, source_rel_path) try: if op_type == "created": @@ -480,14 +517,12 @@ def process_delta_bundle(workspace_path: str, bundle_path: Path, manifest: Dict[ operations_count["failed"] += 1 # Remove original source file if provided - source_rel_path = operation.get("source_path") or operation.get("source_relative_path") - if source_rel_path: - source_path = workspace / source_rel_path - if source_path.exists(): + if safe_source_path is not None and source_rel_path: + if safe_source_path.exists(): try: - source_path.unlink() + safe_source_path.unlink() operations_count["deleted"] += 1 - _cleanup_empty_dirs(source_path.parent, workspace) + _cleanup_empty_dirs(safe_source_path.parent, workspace) except Exception as del_err: logger.error(f"Error deleting source file for move {source_rel_path}: {del_err}") diff --git a/tests/test_upload_service_path_traversal.py b/tests/test_upload_service_path_traversal.py new file mode 100644 index 00000000..4703c2b2 --- /dev/null +++ b/tests/test_upload_service_path_traversal.py @@ -0,0 +1,84 @@ +import io +import json +import tarfile +from pathlib import Path + +import pytest + + +def _write_bundle(tmp_path: Path, operations: list[dict]) -> Path: + bundle_path = tmp_path / "bundle.tar.gz" + payload = json.dumps({"operations": operations}).encode("utf-8") + + with tarfile.open(bundle_path, "w:gz") as tar: + info = tarfile.TarInfo(name="metadata/operations.json") + info.size = len(payload) + tar.addfile(info, io.BytesIO(payload)) + + return bundle_path + + +def test_process_delta_bundle_rejects_traversal_created(tmp_path, monkeypatch): + import scripts.upload_service as us + + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr(us, "WORK_DIR", str(work_dir)) + + bundle = _write_bundle( + tmp_path, + [{"operation": "created", "path": "../../evil.txt"}], + ) + + with pytest.raises(ValueError, match="escapes workspace"): + us.process_delta_bundle( + workspace_path="/home/user/repo", + bundle_path=bundle, + manifest={"bundle_id": "b1"}, + ) + + +def test_process_delta_bundle_rejects_absolute_paths(tmp_path, monkeypatch): + import scripts.upload_service as us + + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr(us, "WORK_DIR", str(work_dir)) + + bundle = _write_bundle( + tmp_path, + [{"operation": "created", "path": "/etc/passwd"}], + ) + + with pytest.raises(ValueError, match="Absolute paths"): + us.process_delta_bundle( + workspace_path="/home/user/repo", + bundle_path=bundle, + manifest={"bundle_id": "b1"}, + ) + + +def test_process_delta_bundle_rejects_traversal_moved_source(tmp_path, monkeypatch): + import scripts.upload_service as us + + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr(us, "WORK_DIR", str(work_dir)) + + bundle = _write_bundle( + tmp_path, + [ + { + "operation": "moved", + "path": "dst.txt", + "source_path": "../../escape.txt", + } + ], + ) + + with pytest.raises(ValueError, match="escapes workspace"): + us.process_delta_bundle( + workspace_path="/home/user/repo", + bundle_path=bundle, + manifest={"bundle_id": "b1"}, + ) From f5826df7fd180d7da80fbf8d18c1811dc23928c5 Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 22:43:36 +0000 Subject: [PATCH 54/55] mcp bridge: improve path mapping and workspace-relative handling - Add remapRelatedPathToClient function for better related_paths processing - Enhance remapHitPaths to handle nested related_paths per result - Update remapStringPath to support workspace-relative path overrides - Simplify payload processing by removing redundant code paths - Bump bridge package version to 0.0.8 Improves path consistency between server and client workspaces with better relative path resolution and environment-based overrides. --- ctx-mcp-bridge/package.json | 2 +- ctx-mcp-bridge/src/resultPathMapping.js | 86 +++++++++++++++++++++---- 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/ctx-mcp-bridge/package.json b/ctx-mcp-bridge/package.json index fc57e3d9..0c15d48e 100644 --- a/ctx-mcp-bridge/package.json +++ b/ctx-mcp-bridge/package.json @@ -1,6 +1,6 @@ { "name": "@context-engine-bridge/context-engine-mcp-bridge", - "version": "0.0.7", + "version": "0.0.8", "description": "Context Engine MCP bridge (http/stdio proxy combining indexer + memory servers)", "bin": { "ctxce": "bin/ctxce.js", diff --git a/ctx-mcp-bridge/src/resultPathMapping.js b/ctx-mcp-bridge/src/resultPathMapping.js index 42f2972e..97f30ce3 100644 --- a/ctx-mcp-bridge/src/resultPathMapping.js +++ b/ctx-mcp-bridge/src/resultPathMapping.js @@ -55,6 +55,49 @@ function computeWorkspaceRelativePath(containerPath, hostPath) { } } +function remapRelatedPathToClient(p, workspaceRoot) { + try { + const s = typeof p === "string" ? p : ""; + const root = typeof workspaceRoot === "string" ? workspaceRoot : ""; + if (!s || !root) { + return p; + } + + const sNorm = s.replace(/\\/g, path.sep); + if (sNorm.startsWith(root + path.sep) || sNorm === root) { + return sNorm; + } + + if (s.startsWith("/work/")) { + const rest = s.slice("/work/".length); + const parts = rest.split("/").filter(Boolean); + if (parts.length >= 2) { + const rel = parts.slice(1).join("/"); + const relNative = _posixToNative(rel); + return path.join(root, relNative); + } + return p; + } + + // If it's already a relative path, join it to the workspace root. + if (!s.startsWith("/") && !s.includes(":") && !s.includes("\\")) { + const relPosix = s.trim(); + if (relPosix && relPosix !== "." && !relPosix.startsWith("../") && relPosix !== "..") { + const relNative = _posixToNative(relPosix); + const joined = path.join(root, relNative); + const relCheck = path.relative(root, joined); + if (relCheck && !relCheck.startsWith(`..${path.sep}`) && relCheck !== "..") { + return joined; + } + } + } + + return p; + } catch { + return p; + } +} + function remapHitPaths(hit, workspaceRoot) { if (!hit || typeof hit !== "object") { return hit; @@ -66,6 +109,14 @@ function remapHitPaths(hit, workspaceRoot) { if (relPath) { out.rel_path = relPath; } + // Remap related_paths nested under each hit (repo_search/hybrid_search emit this per result). + try { + if (Array.isArray(out.related_paths)) { + out.related_paths = out.related_paths.map((p) => remapRelatedPathToClient(p, workspaceRoot)); + } + } catch { + // ignore + } if (workspaceRoot && relPath) { try { const relNative = _posixToNative(relPath); @@ -113,12 +164,32 @@ function remapHitPaths(hit, workspaceRoot) { return out; } -function remapStringPath(p) { +function remapStringPath(p, workspaceRoot) { try { const s = typeof p === "string" ? p : ""; if (!s) { return p; } + // If this is already a path within the current client workspace, rewrite to a + // workspace-relative string when override is enabled. + try { + const root = typeof workspaceRoot === "string" ? workspaceRoot : ""; + if (root) { + const sNorm = s.replace(/\\/g, path.sep); + if (sNorm.startsWith(root + path.sep) || sNorm === root) { + const relNative = path.relative(root, sNorm); + const relPosix = String(relNative).split(path.sep).join("/"); + if (relPosix && !relPosix.startsWith("../") && relPosix !== "..") { + const override = envTruthy(process.env.CTXCE_BRIDGE_OVERRIDE_PATH, true); + if (override) { + return relPosix; + } + } + } + } + } catch { + // ignore + } if (s.startsWith("/work/")) { const rest = s.slice("/work/".length); const parts = rest.split("/").filter(Boolean); @@ -191,18 +262,7 @@ function applyPathMappingToPayload(payload, workspaceRoot) { out.citations = mapHitsArray(out.citations); } if (Array.isArray(out.related_paths)) { - out.related_paths = out.related_paths.map((p) => remapStringPath(p)); - } - - // context_search: {results:[{source:"code"|"memory", ...}]} - if (Array.isArray(out.results)) { - out.results = out.results.map((r) => { - if (!r || typeof r !== "object") { - return r; - } - // Only code results have path-like fields - return remapHitPaths(r, workspaceRoot); - }); + out.related_paths = out.related_paths.map((p) => remapRelatedPathToClient(p, workspaceRoot)); } // Some tools nest under {result:{...}} From 2f237042005d6aa4495b7ff4823d8234cbb09077 Mon Sep 17 00:00:00 2001 From: Reese Date: Fri, 12 Dec 2025 23:51:27 +0000 Subject: [PATCH 55/55] test(ci): avoid FastAPI dependency by extracting delta bundle processing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move process_delta_bundle/get_workspace_key into scripts/upload_delta_bundle so unit tests don’t need to import scripts/upload_service (and therefore don’t require fastapi in CI). Update upload_service to delegate to the new module and adjust traversal tests accordingly. --- scripts/upload_delta_bundle.py | 256 ++++++++++++++++++++ scripts/upload_service.py | 247 +------------------ tests/test_upload_service_path_traversal.py | 6 +- 3 files changed, 262 insertions(+), 247 deletions(-) create mode 100644 scripts/upload_delta_bundle.py diff --git a/scripts/upload_delta_bundle.py b/scripts/upload_delta_bundle.py new file mode 100644 index 00000000..16415aac --- /dev/null +++ b/scripts/upload_delta_bundle.py @@ -0,0 +1,256 @@ +import os +import json +import tarfile +import hashlib +import logging +from pathlib import Path +from typing import Dict, Any + + +try: + from scripts.workspace_state import _extract_repo_name_from_path +except ImportError: + _extract_repo_name_from_path = None + + +logger = logging.getLogger(__name__) + +WORK_DIR = os.environ.get("WORK_DIR", "/work") + + +def get_workspace_key(workspace_path: str) -> str: + """Generate 16-char hash for collision avoidance in remote uploads. + + Remote uploads may have identical folder names from different users, + so uses longer hash than local indexing (8-chars) to ensure uniqueness. + + Both host paths (/home/user/project/repo) and container paths (/work/repo) + should generate the same key for the same repository. + """ + repo_name = Path(workspace_path).name + return hashlib.sha256(repo_name.encode("utf-8")).hexdigest()[:16] + + +def _cleanup_empty_dirs(path: Path, stop_at: Path) -> None: + """Recursively remove empty directories up to stop_at (exclusive).""" + try: + path = path.resolve() + stop_at = stop_at.resolve() + except Exception: + pass + while True: + try: + if path == stop_at or not path.exists() or not path.is_dir(): + break + if any(path.iterdir()): + break + path.rmdir() + path = path.parent + except Exception: + break + + +def process_delta_bundle(workspace_path: str, bundle_path: Path, manifest: Dict[str, Any]) -> Dict[str, int]: + """Process delta bundle and return operation counts.""" + operations_count = { + "created": 0, + "updated": 0, + "deleted": 0, + "moved": 0, + "skipped": 0, + "failed": 0, + } + + try: + # CRITICAL: Always materialize writes under WORK_DIR using a slugged repo directory. + # Do NOT write directly into the client-supplied workspace_path, since that may be a host + # path (e.g. /home/user/repo) that is not mounted/visible to the watcher/indexer. + if _extract_repo_name_from_path: + repo_name = _extract_repo_name_from_path(workspace_path) + if not repo_name: + repo_name = Path(workspace_path).name + else: + repo_name = Path(workspace_path).name + + # Workspace slug: -<16charhash>. This ensures uniqueness across users/workspaces + # that may share the same leaf folder name. + workspace_key = get_workspace_key(workspace_path) + workspace = Path(WORK_DIR) / f"{repo_name}-{workspace_key}" + workspace.mkdir(parents=True, exist_ok=True) + slug_repo_name = f"{repo_name}-{workspace_key}" + + workspace_root = workspace.resolve() + + def _safe_join(base: Path, rel: str) -> Path: + # SECURITY: Prevent path traversal / absolute-path writes by ensuring the resolved + # candidate path stays within the intended workspace root. + rp = Path(str(rel)) + if str(rp) in {".", ""}: + raise ValueError("Invalid operation path") + if rp.is_absolute(): + raise ValueError(f"Absolute paths are not allowed: {rel}") + base_resolved = base.resolve() + candidate = (base_resolved / rp).resolve() + try: + ok = candidate.is_relative_to(base_resolved) + except Exception: + ok = os.path.commonpath([str(base_resolved), str(candidate)]) == str(base_resolved) + if not ok: + raise ValueError(f"Path escapes workspace: {rel}") + return candidate + + with tarfile.open(bundle_path, "r:gz") as tar: + ops_member = None + for member in tar.getnames(): + if member.endswith("metadata/operations.json"): + ops_member = member + break + + if not ops_member: + raise ValueError("operations.json not found in bundle") + + ops_file = tar.extractfile(ops_member) + if not ops_file: + raise ValueError("Cannot extract operations.json") + + operations_data = json.loads(ops_file.read().decode("utf-8")) + operations = operations_data.get("operations", []) + + # Best-effort: extract git history metadata for watcher to ingest + try: + git_member = None + for member in tar.getnames(): + if member.endswith("metadata/git_history.json"): + git_member = member + break + if git_member: + git_file = tar.extractfile(git_member) + if git_file: + history_bytes = git_file.read() + history_dir = workspace / ".remote-git" + history_dir.mkdir(parents=True, exist_ok=True) + bundle_id = manifest.get("bundle_id") or "unknown" + history_path = history_dir / f"git_history_{bundle_id}.json" + try: + history_path.write_bytes(history_bytes) + except Exception as write_err: + logger.debug( + f"[upload_service] Failed to write git history manifest: {write_err}", + ) + except Exception as git_err: + logger.debug(f"[upload_service] Error extracting git history metadata: {git_err}") + + for operation in operations: + op_type = operation.get("operation") + rel_path = operation.get("path") + + if not rel_path: + operations_count["skipped"] += 1 + continue + + # Defensive guard: if the operation path already includes the slugged repo name + # ("-/..."), then writing it under workspace_root would create + # a nested slug directory ("slug/slug/..."), which is almost always client misuse. + if rel_path == slug_repo_name or rel_path.startswith(slug_repo_name + "/"): + msg = ( + f"[upload_service] Refusing to apply operation {op_type} for suspicious path {rel_path} " + f"which already contains workspace slug {slug_repo_name}" + ) + logger.error(msg) + raise ValueError(msg) + + target_path = _safe_join(workspace_root, rel_path) + + safe_source_path = None + source_rel_path = None + if op_type == "moved": + source_rel_path = operation.get("source_path") or operation.get("source_relative_path") + if source_rel_path: + safe_source_path = _safe_join(workspace_root, source_rel_path) + + try: + if op_type == "created": + file_member = None + for member in tar.getnames(): + if member.endswith(f"files/created/{rel_path}"): + file_member = member + break + + if file_member: + file_content = tar.extractfile(file_member) + if file_content: + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_bytes(file_content.read()) + operations_count["created"] += 1 + else: + operations_count["failed"] += 1 + else: + operations_count["failed"] += 1 + + elif op_type == "updated": + file_member = None + for member in tar.getnames(): + if member.endswith(f"files/updated/{rel_path}"): + file_member = member + break + + if file_member: + file_content = tar.extractfile(file_member) + if file_content: + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_bytes(file_content.read()) + operations_count["updated"] += 1 + else: + operations_count["failed"] += 1 + else: + operations_count["failed"] += 1 + + elif op_type == "moved": + file_member = None + for member in tar.getnames(): + if member.endswith(f"files/moved/{rel_path}"): + file_member = member + break + + if file_member: + file_content = tar.extractfile(file_member) + if file_content: + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_bytes(file_content.read()) + operations_count["moved"] += 1 + else: + operations_count["failed"] += 1 + else: + operations_count["failed"] += 1 + + if safe_source_path is not None and source_rel_path: + if safe_source_path.exists(): + try: + safe_source_path.unlink() + operations_count["deleted"] += 1 + _cleanup_empty_dirs(safe_source_path.parent, workspace) + except Exception as del_err: + logger.error( + f"Error deleting source file for move {source_rel_path}: {del_err}", + ) + + elif op_type == "deleted": + if target_path.exists(): + target_path.unlink() + _cleanup_empty_dirs(target_path.parent, workspace) + operations_count["deleted"] += 1 + else: + operations_count["skipped"] += 1 + + else: + operations_count["skipped"] += 1 + + except Exception as e: + logger.error(f"Error processing operation {op_type} for {rel_path}: {e}") + operations_count["failed"] += 1 + + return operations_count + + except Exception as e: + logger.error(f"Error processing delta bundle: {e}") + raise diff --git a/scripts/upload_service.py b/scripts/upload_service.py index 36255f07..0e179138 100644 --- a/scripts/upload_service.py +++ b/scripts/upload_service.py @@ -10,7 +10,6 @@ import json import tarfile import tempfile -import hashlib import asyncio import logging from pathlib import Path @@ -21,6 +20,9 @@ from fastapi import FastAPI, File, UploadFile, Form, HTTPException, Request, status from fastapi.responses import JSONResponse, RedirectResponse from fastapi.middleware.cors import CORSMiddleware + +from scripts.upload_delta_bundle import get_workspace_key, process_delta_bundle + from pydantic import BaseModel, Field from scripts.auth_backend import ( AuthDisabledError, @@ -253,18 +255,6 @@ def _require_admin_session(request: Request) -> Dict[str, Any]: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin required") return record -def get_workspace_key(workspace_path: str) -> str: - """Generate 16-char hash for collision avoidance in remote uploads. - - Remote uploads may have identical folder names from different users, - so uses longer hash than local indexing (8-chars) to ensure uniqueness. - - Both host paths (/home/user/project/repo) and container paths (/work/repo) - should generate the same key for the same repository. - """ - repo_name = Path(workspace_path).name - return hashlib.sha256(repo_name.encode('utf-8')).hexdigest()[:16] - def get_next_sequence(workspace_path: str) -> int: """Get next sequence number for workspace.""" key = get_workspace_key(workspace_path) @@ -317,237 +307,6 @@ def validate_bundle_format(bundle_path: Path) -> Dict[str, Any]: except Exception as e: raise ValueError(f"Invalid bundle format: {str(e)}") -def _cleanup_empty_dirs(path: Path, stop_at: Path) -> None: - """Recursively remove empty directories up to stop_at (exclusive).""" - try: - path = path.resolve() - stop_at = stop_at.resolve() - except Exception: - pass - while True: - try: - if path == stop_at or not path.exists() or not path.is_dir(): - break - if any(path.iterdir()): - break - path.rmdir() - path = path.parent - except Exception: - break - -def process_delta_bundle(workspace_path: str, bundle_path: Path, manifest: Dict[str, Any]) -> Dict[str, int]: - """Process delta bundle and return operation counts.""" - operations_count = { - "created": 0, - "updated": 0, - "deleted": 0, - "moved": 0, - "skipped": 0, - "failed": 0 - } - - try: - # CRITICAL FIX: Extract repo name and create workspace under WORK_DIR - # Previous bug: used source workspace_path directly, extracting files outside /work - # This caused watcher service to never see uploaded files - if _extract_repo_name_from_path: - repo_name = _extract_repo_name_from_path(workspace_path) - # Fallback to directory name if repo detection fails - if not repo_name: - repo_name = Path(workspace_path).name - else: - # Fallback: use directory name - repo_name = Path(workspace_path).name - - # Generate workspace under WORK_DIR using repo name hash - workspace_key = get_workspace_key(workspace_path) - workspace = Path(WORK_DIR) / f"{repo_name}-{workspace_key}" - workspace.mkdir(parents=True, exist_ok=True) - slug_repo_name = f"{repo_name}-{workspace_key}" - - workspace_root = workspace.resolve() - - def _safe_join(base: Path, rel: str) -> Path: - rp = Path(str(rel)) - if str(rp) in {".", ""}: - raise ValueError("Invalid operation path") - if rp.is_absolute(): - raise ValueError(f"Absolute paths are not allowed: {rel}") - base_resolved = base.resolve() - candidate = (base_resolved / rp).resolve() - try: - ok = candidate.is_relative_to(base_resolved) - except Exception: - ok = os.path.commonpath([str(base_resolved), str(candidate)]) == str(base_resolved) - if not ok: - raise ValueError(f"Path escapes workspace: {rel}") - return candidate - - with tarfile.open(bundle_path, "r:gz") as tar: - # Extract operations metadata - ops_member = None - for member in tar.getnames(): - if member.endswith("metadata/operations.json"): - ops_member = member - break - - if not ops_member: - raise ValueError("operations.json not found in bundle") - - ops_file = tar.extractfile(ops_member) - if not ops_file: - raise ValueError("Cannot extract operations.json") - - operations_data = json.loads(ops_file.read().decode('utf-8')) - operations = operations_data.get("operations", []) - - # Best-effort: extract git history metadata for watcher to ingest - try: - git_member = None - for member in tar.getnames(): - if member.endswith("metadata/git_history.json"): - git_member = member - break - if git_member: - git_file = tar.extractfile(git_member) - if git_file: - history_bytes = git_file.read() - history_dir = workspace / ".remote-git" - history_dir.mkdir(parents=True, exist_ok=True) - bundle_id = manifest.get("bundle_id") or "unknown" - history_path = history_dir / f"git_history_{bundle_id}.json" - try: - history_path.write_bytes(history_bytes) - except Exception as write_err: - logger.debug(f"[upload_service] Failed to write git history manifest: {write_err}") - except Exception as git_err: - logger.debug(f"[upload_service] Error extracting git history metadata: {git_err}") - - # Process each operation - for operation in operations: - op_type = operation.get("operation") - rel_path = operation.get("path") - - if not rel_path: - operations_count["skipped"] += 1 - continue - - # Defensive guard: if rel_path already starts with the slugged - # repo name (e.g. "-/"), writing to workspace / rel_path - # would create a nested slug directory ("slug/slug/..."). This is - # almost certainly a misconfigured client using a dev-workspace - # mirror as the workspace root. Treat this as a hard error so the - # bundle does not silently create recursive structures. - # TODO: http error code/msg for extension toast? - if rel_path == slug_repo_name or rel_path.startswith(slug_repo_name + "/"): - msg = ( - f"[upload_service] Refusing to apply operation {op_type} for suspicious path {rel_path} " - f"which already contains workspace slug {slug_repo_name}" - ) - logger.error(msg) - raise ValueError(msg) - - target_path = _safe_join(workspace_root, rel_path) - - safe_source_path = None - source_rel_path = None - if op_type == "moved": - source_rel_path = operation.get("source_path") or operation.get( - "source_relative_path" - ) - if source_rel_path: - safe_source_path = _safe_join(workspace_root, source_rel_path) - - try: - if op_type == "created": - # Extract file from bundle - file_member = None - for member in tar.getnames(): - if member.endswith(f"files/created/{rel_path}"): - file_member = member - break - - if file_member: - file_content = tar.extractfile(file_member) - if file_content: - target_path.parent.mkdir(parents=True, exist_ok=True) - target_path.write_bytes(file_content.read()) - operations_count["created"] += 1 - else: - operations_count["failed"] += 1 - else: - operations_count["failed"] += 1 - - elif op_type == "updated": - # Extract updated file - file_member = None - for member in tar.getnames(): - if member.endswith(f"files/updated/{rel_path}"): - file_member = member - break - - if file_member: - file_content = tar.extractfile(file_member) - if file_content: - target_path.parent.mkdir(parents=True, exist_ok=True) - target_path.write_bytes(file_content.read()) - operations_count["updated"] += 1 - else: - operations_count["failed"] += 1 - else: - operations_count["failed"] += 1 - - elif op_type == "moved": - # Extract moved file to destination - file_member = None - for member in tar.getnames(): - if member.endswith(f"files/moved/{rel_path}"): - file_member = member - break - - if file_member: - file_content = tar.extractfile(file_member) - if file_content: - target_path.parent.mkdir(parents=True, exist_ok=True) - target_path.write_bytes(file_content.read()) - operations_count["moved"] += 1 - else: - operations_count["failed"] += 1 - else: - operations_count["failed"] += 1 - - # Remove original source file if provided - if safe_source_path is not None and source_rel_path: - if safe_source_path.exists(): - try: - safe_source_path.unlink() - operations_count["deleted"] += 1 - _cleanup_empty_dirs(safe_source_path.parent, workspace) - except Exception as del_err: - logger.error(f"Error deleting source file for move {source_rel_path}: {del_err}") - - elif op_type == "deleted": - # Delete file - if target_path.exists(): - target_path.unlink() - _cleanup_empty_dirs(target_path.parent, workspace) - operations_count["deleted"] += 1 - else: - operations_count["skipped"] += 1 - - else: - operations_count["skipped"] += 1 - - except Exception as e: - logger.error(f"Error processing operation {op_type} for {rel_path}: {e}") - operations_count["failed"] += 1 - - return operations_count - - except Exception as e: - logger.error(f"Error processing delta bundle: {e}") - raise - async def _process_bundle_background( workspace_path: str, diff --git a/tests/test_upload_service_path_traversal.py b/tests/test_upload_service_path_traversal.py index 4703c2b2..2c9ba889 100644 --- a/tests/test_upload_service_path_traversal.py +++ b/tests/test_upload_service_path_traversal.py @@ -19,7 +19,7 @@ def _write_bundle(tmp_path: Path, operations: list[dict]) -> Path: def test_process_delta_bundle_rejects_traversal_created(tmp_path, monkeypatch): - import scripts.upload_service as us + import scripts.upload_delta_bundle as us work_dir = tmp_path / "work" work_dir.mkdir(parents=True, exist_ok=True) @@ -39,7 +39,7 @@ def test_process_delta_bundle_rejects_traversal_created(tmp_path, monkeypatch): def test_process_delta_bundle_rejects_absolute_paths(tmp_path, monkeypatch): - import scripts.upload_service as us + import scripts.upload_delta_bundle as us work_dir = tmp_path / "work" work_dir.mkdir(parents=True, exist_ok=True) @@ -59,7 +59,7 @@ def test_process_delta_bundle_rejects_absolute_paths(tmp_path, monkeypatch): def test_process_delta_bundle_rejects_traversal_moved_source(tmp_path, monkeypatch): - import scripts.upload_service as us + import scripts.upload_delta_bundle as us work_dir = tmp_path / "work" work_dir.mkdir(parents=True, exist_ok=True)