diff --git a/CHANGELOG.md b/CHANGELOG.md index b92ed0f..1a4f2cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,7 @@ Model Context Protocol server for AI assistant integration: #### Portable Package Format (`semanticwiki pack/unpack`) Bundle wikis with RAG indexes for sharing: -- `.archiwiki` compressed format with manifest +- `.semantics` compressed format with manifest - Includes wiki content + embeddings + BM25 index - Extract wiki-only or full package diff --git a/README.md b/README.md index dd01603..3db0a18 100644 --- a/README.md +++ b/README.md @@ -178,13 +178,13 @@ Create portable wiki packages for sharing: ```bash # Create package (includes wiki + RAG index) -semanticwiki pack -w ./wiki -o ./my-wiki.archiwiki +semanticwiki pack -w ./wiki -o ./my-wiki.semantics # Extract package -semanticwiki unpack -p ./my-wiki.archiwiki -o ./extracted +semanticwiki unpack -p ./my-wiki.semantics -o ./extracted # Extract wiki only (no RAG index) -semanticwiki unpack -p ./my-wiki.archiwiki -o ./extracted --wiki-only +semanticwiki unpack -p ./my-wiki.semantics -o ./extracted --wiki-only ``` ### Large Codebase Options @@ -287,7 +287,7 @@ semanticwiki generate -r ./my-project --max-turns 50 | Option | Description | Default | |--------|-------------|---------| | `-w, --wiki ` | Wiki directory (required) | - | -| `-o, --output ` | Output package path | `.archiwiki` | +| `-o, --output ` | Output package path | `.semantics` | | `-n, --name ` | Package name | wiki folder name | | `--no-rag` | Exclude RAG index | - | diff --git a/package-lock.json b/package-lock.json index c5caf49..51a207b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,6 @@ "cheerio": "^1.0.0-rc.12", "commander": "^12.0.0", "dotenv": "^16.3.1", - "faiss-node": "^0.5.1", "glob": "^10.3.10", "gray-matter": "^4.0.3", "inquirer": "^9.2.12", @@ -44,12 +43,15 @@ }, "engines": { "node": ">=18.0.0" + }, + "optionalDependencies": { + "faiss-node": "^0.5.1" } }, "node_modules/@anthropic-ai/claude-agent-sdk": { - "version": "0.1.75", - "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk/-/claude-agent-sdk-0.1.75.tgz", - "integrity": "sha512-8iYosqTq98pm6Z1IJ0OY50mpkSZ7fdSO8cJJjFHXiKHCwfmiPZ05vLYUed2p2an9GPLpXtJhcGDfrgseenrgPw==", + "version": "0.1.77", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk/-/claude-agent-sdk-0.1.77.tgz", + "integrity": "sha512-ZEjWQtkoB2MEY6K16DWMmF+8OhywAynH0m08V265cerbZ8xPD/2Ng2jPzbbO40mPeFSsMDJboShL+a3aObP0Jg==", "license": "SEE LICENSE IN README.md", "engines": { "node": ">=18.0.0" @@ -65,7 +67,7 @@ "@img/sharp-win32-x64": "^0.33.5" }, "peerDependencies": { - "zod": "^3.24.1 || ^4.0.0" + "zod": "^3.25.0 || ^4.0.0" } }, "node_modules/@anthropic-ai/sdk": { @@ -109,13 +111,13 @@ } }, "node_modules/@babel/parser": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.5.tgz", - "integrity": "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.6.tgz", + "integrity": "sha512-TeR9zWR18BvbfPmGbLampPMW+uW1NZnJlRuuHso8i87QZNq2JRF9i6RgxRqtEq+wQGsS19NNTWr2duhnE49mfQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.28.5" + "@babel/types": "^7.28.6" }, "bin": { "parser": "bin/babel-parser.js" @@ -125,18 +127,18 @@ } }, "node_modules/@babel/runtime": { - "version": "7.28.4", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.4.tgz", - "integrity": "sha512-Q/N6JNWvIvPnLDvjlE1OUBLPQHH6l3CltCEsHIujp45zQUSSh8K+gHnaEX45yAT1nyngnINhvWtzN+Nb9D8RAQ==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz", + "integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==", "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/types": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.5.tgz", - "integrity": "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.6.tgz", + "integrity": "sha512-0ZrskXVEHSWIqZM/sQZ4EV3jZJXRkio/WCxaqKZP1g//CEWEPSfeZFcms4XeKBCHU0ZKnIkdJeU/kF+eRp5lBg==", "dev": true, "license": "MIT", "dependencies": { @@ -610,9 +612,9 @@ } }, "node_modules/@hono/node-server": { - "version": "1.19.7", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.7.tgz", - "integrity": "sha512-vUcD0uauS7EU2caukW8z5lJKtoGMokxNbJtBiwHgpqxEXokaHCBkQUmCHhjFB1VUTWdqj25QoMkMKzgjq+uhrw==", + "version": "1.19.9", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.9.tgz", + "integrity": "sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw==", "license": "MIT", "engines": { "node": ">=18.14.1" @@ -1129,9 +1131,9 @@ } }, "node_modules/@inquirer/external-editor/node_modules/iconv-lite": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.1.tgz", - "integrity": "sha512-2Tth85cXwGFHfvRgZWszZSvdo+0Xsqmw8k8ZwxScfcBneNUraK+dxRxRm24nszx80Y0TVio8kKLt5sLE7ZCLlw==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" @@ -1203,18 +1205,6 @@ "url": "https://github.com/chalk/ansi-regex?sponsor=1" } }, - "node_modules/@isaacs/cliui/node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, "node_modules/@isaacs/cliui/node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", @@ -1341,9 +1331,9 @@ } }, "node_modules/@modelcontextprotocol/sdk": { - "version": "1.25.1", - "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.25.1.tgz", - "integrity": "sha512-yO28oVFFC7EBoiKdAn+VqRm+plcfv4v0xp6osG/VsCB0NlPZWi87ajbCZZ8f/RvOFLEu7//rSRmuZZ7lMoe3gQ==", + "version": "1.25.2", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.25.2.tgz", + "integrity": "sha512-LZFeo4F9M5qOhC/Uc1aQSrBHxMrvxett+9KLHt7OhcExtoiRN9DKgbZffMP/nxjutWDQpfMDfP3nkHI4X9ijww==", "license": "MIT", "dependencies": { "@hono/node-server": "^1.19.7", @@ -1395,21 +1385,6 @@ "mcp-server-filesystem": "dist/index.js" } }, - "node_modules/@modelcontextprotocol/server-filesystem/node_modules/minimatch": { - "version": "10.1.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", - "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/@node-llama-cpp/linux-arm64": { "version": "3.15.0", "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-arm64/-/linux-arm64-3.15.0.tgz", @@ -2598,9 +2573,9 @@ } }, "node_modules/@types/node": { - "version": "20.19.28", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.28.tgz", - "integrity": "sha512-VyKBr25BuFDzBFCK5sUM6ZXiWfqgCTwTAOK8qzGV/m9FCirXYDlmczJ+d5dXBAQALGCdRRdbteKYfJ84NGEusw==", + "version": "20.19.30", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.30.tgz", + "integrity": "sha512-WJtwWJu7UdlvzEAUm484QNg5eAoq5QR08KDNx7g45Usrs2NtOPiX8ugDqmKdXkyL03rBqU5dYNYVQetEpBHq2g==", "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -2627,18 +2602,17 @@ } }, "node_modules/@vitest/coverage-v8": { - "version": "4.0.16", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.0.16.tgz", - "integrity": "sha512-2rNdjEIsPRzsdu6/9Eq0AYAzYdpP6Bx9cje9tL3FE5XzXRQF1fNU9pe/1yE8fCrS0HD+fBtt6gLPh6LI57tX7A==", + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.0.17.tgz", + "integrity": "sha512-/6zU2FLGg0jsd+ePZcwHRy3+WpNTBBhDY56P4JTRqUN/Dp6CvOEa9HrikcQ4KfV2b2kAHUFB4dl1SuocWXSFEw==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.0.16", - "ast-v8-to-istanbul": "^0.3.8", + "@vitest/utils": "4.0.17", + "ast-v8-to-istanbul": "^0.3.10", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", - "istanbul-lib-source-maps": "^5.0.6", "istanbul-reports": "^3.2.0", "magicast": "^0.5.1", "obug": "^2.1.1", @@ -2649,8 +2623,8 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/browser": "4.0.16", - "vitest": "4.0.16" + "@vitest/browser": "4.0.17", + "vitest": "4.0.17" }, "peerDependenciesMeta": { "@vitest/browser": { @@ -2659,16 +2633,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.0.16", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.16.tgz", - "integrity": "sha512-eshqULT2It7McaJkQGLkPjPjNph+uevROGuIMJdG3V+0BSR2w9u6J9Lwu+E8cK5TETlfou8GRijhafIMhXsimA==", + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.17.tgz", + "integrity": "sha512-mEoqP3RqhKlbmUmntNDDCJeTDavDR+fVYkSOw8qRwJFaW/0/5zA9zFeTrHqNtcmwh6j26yMmwx2PqUDPzt5ZAQ==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.0.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.0.16", - "@vitest/utils": "4.0.16", + "@vitest/spy": "4.0.17", + "@vitest/utils": "4.0.17", "chai": "^6.2.1", "tinyrainbow": "^3.0.3" }, @@ -2677,13 +2651,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.0.16", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.0.16.tgz", - "integrity": "sha512-yb6k4AZxJTB+q9ycAvsoxGn+j/po0UaPgajllBgt1PzoMAAmJGYFdDk0uCcRcxb3BrME34I6u8gHZTQlkqSZpg==", + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.0.17.tgz", + "integrity": "sha512-+ZtQhLA3lDh1tI2wxe3yMsGzbp7uuJSWBM1iTIKCbppWTSBN09PUC+L+fyNlQApQoR+Ps8twt2pbSSXg2fQVEQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.0.16", + "@vitest/spy": "4.0.17", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -2704,9 +2678,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.0.16", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.0.16.tgz", - "integrity": "sha512-eNCYNsSty9xJKi/UdVD8Ou16alu7AYiS2fCPRs0b1OdhJiV89buAXQLpTbe+X8V9L6qrs9CqyvU7OaAopJYPsA==", + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.0.17.tgz", + "integrity": "sha512-Ah3VAYmjcEdHg6+MwFE17qyLqBHZ+ni2ScKCiW2XrlSBV4H3Z7vYfPfz7CWQ33gyu76oc0Ai36+kgLU3rfF4nw==", "dev": true, "license": "MIT", "dependencies": { @@ -2717,13 +2691,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.0.16", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.0.16.tgz", - "integrity": "sha512-VWEDm5Wv9xEo80ctjORcTQRJ539EGPB3Pb9ApvVRAY1U/WkHXmmYISqU5E79uCwcW7xYUV38gwZD+RV755fu3Q==", + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.0.17.tgz", + "integrity": "sha512-JmuQyf8aMWoo/LmNFppdpkfRVHJcsgzkbCA+/Bk7VfNH7RE6Ut2qxegeyx2j3ojtJtKIbIGy3h+KxGfYfk28YQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.0.16", + "@vitest/utils": "4.0.17", "pathe": "^2.0.3" }, "funding": { @@ -2731,13 +2705,13 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.0.16", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.0.16.tgz", - "integrity": "sha512-sf6NcrYhYBsSYefxnry+DR8n3UV4xWZwWxYbCJUt2YdvtqzSPR7VfGrY0zsv090DAbjFZsi7ZaMi1KnSRyK1XA==", + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.0.17.tgz", + "integrity": "sha512-npPelD7oyL+YQM2gbIYvlavlMVWUfNNGZPcu0aEUQXt7FXTuqhmgiYupPnAanhKvyP6Srs2pIbWo30K0RbDtRQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.0.16", + "@vitest/pretty-format": "4.0.17", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -2746,9 +2720,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.0.16", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.0.16.tgz", - "integrity": "sha512-4jIOWjKP0ZUaEmJm00E0cOBLU+5WE0BpeNr3XN6TEF05ltro6NJqHWxXD0kA8/Zc8Nh23AT8WQxwNG+WeROupw==", + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.0.17.tgz", + "integrity": "sha512-I1bQo8QaP6tZlTomQNWKJE6ym4SHf3oLS7ceNjozxxgzavRAgZDc06T7kD8gb9bXKEgcLNt00Z+kZO6KaJ62Ew==", "dev": true, "license": "MIT", "funding": { @@ -2756,13 +2730,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.0.16", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.0.16.tgz", - "integrity": "sha512-h8z9yYhV3e1LEfaQ3zdypIrnAg/9hguReGZoS7Gl0aBG5xgA410zBqECqmaF/+RkTggRsfnzc1XaAHA6bmUufA==", + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.0.17.tgz", + "integrity": "sha512-RG6iy+IzQpa9SB8HAFHJ9Y+pTzI+h8553MrciN9eC6TFBErqrQaTas4vG+MVj8S4uKk8uTT2p0vgZPnTdxd96w==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.0.16", + "@vitest/pretty-format": "4.0.17", "tinyrainbow": "^3.0.3" }, "funding": { @@ -2782,31 +2756,6 @@ "node": ">= 0.6" } }, - "node_modules/accepts/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/accepts/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/ajv": { "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", @@ -2865,15 +2814,12 @@ } }, "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, "engines": { - "node": ">=8" + "node": ">=12" }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" @@ -3020,9 +2966,9 @@ } }, "node_modules/body-parser": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.1.tgz", - "integrity": "sha512-nfDwkulwiZYQIGwxdy0RUmowMhKcFVcYXUU7m4QlKYim1rUtg83xm2yjZ40QjDuc291AJjjeSc9b++AWHSgSHw==", + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", + "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", "license": "MIT", "dependencies": { "bytes": "^3.1.2", @@ -3031,7 +2977,7 @@ "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", - "qs": "^6.14.0", + "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" }, @@ -3044,9 +2990,9 @@ } }, "node_modules/body-parser/node_modules/iconv-lite": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.1.tgz", - "integrity": "sha512-2Tth85cXwGFHfvRgZWszZSvdo+0Xsqmw8k8ZwxScfcBneNUraK+dxRxRm24nszx80Y0TVio8kKLt5sLE7ZCLlw==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" @@ -3226,10 +3172,13 @@ "license": "MIT" }, "node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "license": "ISC" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "license": "ISC", + "engines": { + "node": ">=10" + } }, "node_modules/ci-info": { "version": "4.3.1", @@ -3296,6 +3245,21 @@ "node": ">=12" } }, + "node_modules/cliui/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/cliui/node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -3348,72 +3312,6 @@ "node": ">= 14.15.0" } }, - "node_modules/cmake-js/node_modules/chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/cmake-js/node_modules/minipass": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", - "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", - "license": "ISC", - "engines": { - "node": ">=8" - } - }, - "node_modules/cmake-js/node_modules/minizlib": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", - "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", - "license": "MIT", - "dependencies": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/cmake-js/node_modules/minizlib/node_modules/minipass": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", - "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cmake-js/node_modules/tar": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", - "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", - "license": "ISC", - "dependencies": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", - "minipass": "^5.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/cmake-js/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "license": "ISC" - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -4112,31 +4010,6 @@ "express": ">= 4.11" } }, - "node_modules/express/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/express/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/extend-shallow": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", @@ -4155,6 +4028,7 @@ "integrity": "sha512-zD8wobJn8C6OLWo68Unho+Ih8l6nSRB2w3Amj01a+xc4bsEvd2mBDLklAn7VocA9XO3WDvQL/bLpi5flkCn/XQ==", "hasInstallScript": true, "license": "MIT", + "optional": true, "dependencies": { "bindings": "^1.5.0", "node-addon-api": "^6.0.0", @@ -4348,6 +4222,27 @@ "node": ">= 6" } }, + "node_modules/form-data/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/form-data/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -4410,12 +4305,6 @@ "node": ">=8" } }, - "node_modules/fs-minipass/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "license": "ISC" - }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -4550,6 +4439,21 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/glob/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/global-agent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", @@ -4689,9 +4593,9 @@ } }, "node_modules/hono": { - "version": "4.11.1", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.11.1.tgz", - "integrity": "sha512-KsFcH0xxHes0J4zaQgWbYwmz3UPOOskdqZmItstUG93+Wk1ePBLkLGwbP9zlmh1BFUiL8Qp+Xfu9P7feJWpGNg==", + "version": "4.11.4", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.11.4.tgz", + "integrity": "sha512-U7tt8JsyrxSRKspfhtLET79pU8K+tInj5QZXs1jSugO1Vq5dFj3kmZsRldo29mTBfcjDRVRXrEZ6LS63Cog9ZA==", "license": "MIT", "peer": true, "engines": { @@ -4872,6 +4776,21 @@ "node": ">=0.12.0" } }, + "node_modules/inquirer/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/inquirer/node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -5134,12 +5053,18 @@ } }, "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", + "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.3.1" + }, "engines": { - "node": ">=8" + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/is-interactive": { @@ -5203,21 +5128,6 @@ "node": ">=10" } }, - "node_modules/istanbul-lib-source-maps": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-5.0.6.tgz", - "integrity": "sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.23", - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/istanbul-reports": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", @@ -5341,14 +5251,26 @@ "license": "MIT" }, "node_modules/log-symbols": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-6.0.0.tgz", - "integrity": "sha512-i24m8rpwhmPIS4zscNzK6MSEhk0DUWa/8iYQWxhffV8jkI4Phvs3F+quL5xvS0gdQR0FyTCMMH33Y78dDTzzIw==", - "license": "MIT", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-7.0.1.tgz", + "integrity": "sha512-ja1E3yCr9i/0hmBVaM0bfwDjnGy8I/s6PP4DFp+yP+a+mrHO4Rm7DtmnqROTUkHIkqffC84YY7AeqX6oFk0WFg==", + "license": "MIT", "dependencies": { - "chalk": "^5.3.0", - "is-unicode-supported": "^1.3.0" + "is-unicode-supported": "^2.0.0", + "yoctocolors": "^2.1.1" + }, + "engines": { + "node": ">=18" }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-symbols/node_modules/is-unicode-supported": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", + "integrity": "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==", + "license": "MIT", "engines": { "node": ">=18" }, @@ -5497,24 +5419,28 @@ } }, "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "license": "MIT", "dependencies": { - "mime-db": "1.52.0" + "mime-db": "^1.54.0" }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/mimic-fn": { @@ -5551,15 +5477,15 @@ } }, "node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "license": "ISC", + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", + "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", + "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^2.0.1" + "@isaacs/brace-expansion": "^5.0.0" }, "engines": { - "node": ">=16 || 14 >=14.17" + "node": "20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -5584,15 +5510,28 @@ } }, "node_modules/minizlib": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", - "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", "license": "MIT", "dependencies": { - "minipass": "^7.1.2" + "minipass": "^3.0.0", + "yallist": "^4.0.0" }, "engines": { - "node": ">= 18" + "node": ">= 8" + } + }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" } }, "node_modules/mkdirp": { @@ -5629,10 +5568,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.11", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", - "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", - "dev": true, + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.6.tgz", + "integrity": "sha512-c7+7RQ+dMB5dPwwCp4ee1/iV/q2P6aK1mTZcfr1BTuVlyW9hJYiMPybJCcnBlQtuSmTIWNeazm/zqNoZSSElBg==", "funding": [ { "type": "github", @@ -5641,10 +5579,10 @@ ], "license": "MIT", "bin": { - "nanoid": "bin/nanoid.cjs" + "nanoid": "bin/nanoid.js" }, "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + "node": "^18 || >=20" } }, "node_modules/napi-build-utils": { @@ -5663,9 +5601,9 @@ } }, "node_modules/node-abi": { - "version": "3.85.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.85.0.tgz", - "integrity": "sha512-zsFhmbkAzwhTft6nd3VxcG0cvJsT70rL+BIGHWVq5fi6MwGrHwzqKaxXE+Hl2GmnGItnDKPPkO5/LQqjVkIdFg==", + "version": "3.86.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.86.0.tgz", + "integrity": "sha512-sn9Et4N3ynsetj3spsZR729DVlGH6iBG4RiDMV7HEp3guyOW6W3S0unGpLDxT50mXortGUMax/ykUNQXdqc/Xg==", "license": "MIT", "dependencies": { "semver": "^7.3.5" @@ -5678,7 +5616,8 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/node-api-headers": { "version": "1.7.0", @@ -5791,40 +5730,6 @@ "node": ">=16" } }, - "node_modules/node-llama-cpp/node_modules/log-symbols": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-7.0.1.tgz", - "integrity": "sha512-ja1E3yCr9i/0hmBVaM0bfwDjnGy8I/s6PP4DFp+yP+a+mrHO4Rm7DtmnqROTUkHIkqffC84YY7AeqX6oFk0WFg==", - "license": "MIT", - "dependencies": { - "is-unicode-supported": "^2.0.0", - "yoctocolors": "^2.1.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/node-llama-cpp/node_modules/nanoid": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.6.tgz", - "integrity": "sha512-c7+7RQ+dMB5dPwwCp4ee1/iV/q2P6aK1mTZcfr1BTuVlyW9hJYiMPybJCcnBlQtuSmTIWNeazm/zqNoZSSElBg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.js" - }, - "engines": { - "node": "^18 || >=20" - } - }, "node_modules/node-llama-cpp/node_modules/node-addon-api": { "version": "8.5.0", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.5.0.tgz", @@ -6023,6 +5928,52 @@ "tar": "^7.0.1" } }, + "node_modules/onnxruntime-node/node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/onnxruntime-node/node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/onnxruntime-node/node_modules/tar": { + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.2.tgz", + "integrity": "sha512-7NyxrTE4Anh8km8iEy7o0QYPs+0JKBTj5ZaqHg6B39erLg0qYXN3BijtShwbsNSvQ+LN75+KV+C4QR/f6Gwnpg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/onnxruntime-node/node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/onnxruntime-web": { "version": "1.22.0-dev.20250409-89f8206ba4", "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.22.0-dev.20250409-89f8206ba4.tgz", @@ -6096,6 +6047,34 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/ora/node_modules/log-symbols": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-6.0.0.tgz", + "integrity": "sha512-i24m8rpwhmPIS4zscNzK6MSEhk0DUWa/8iYQWxhffV8jkI4Phvs3F+quL5xvS0gdQR0FyTCMMH33Y78dDTzzIw==", + "license": "MIT", + "dependencies": { + "chalk": "^5.3.0", + "is-unicode-supported": "^1.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora/node_modules/log-symbols/node_modules/is-unicode-supported": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.3.0.tgz", + "integrity": "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/ora/node_modules/string-width": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", @@ -6309,6 +6288,25 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/postcss/node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -6442,9 +6440,9 @@ } }, "node_modules/qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "version": "6.14.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", + "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", "license": "BSD-3-Clause", "dependencies": { "side-channel": "^1.1.0" @@ -6481,9 +6479,9 @@ } }, "node_modules/raw-body/node_modules/iconv-lite": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.1.tgz", - "integrity": "sha512-2Tth85cXwGFHfvRgZWszZSvdo+0Xsqmw8k8ZwxScfcBneNUraK+dxRxRm24nszx80Y0TVio8kKLt5sLE7ZCLlw==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" @@ -6753,31 +6751,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/send/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/send/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/serialize-error": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", @@ -7359,33 +7332,6 @@ "url": "https://github.com/chalk/slice-ansi?sponsor=1" } }, - "node_modules/slice-ansi/node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/slice-ansi/node_modules/is-fullwidth-code-point": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", - "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", - "license": "MIT", - "dependencies": { - "get-east-asian-width": "^1.3.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -7476,18 +7422,6 @@ "url": "https://github.com/chalk/ansi-regex?sponsor=1" } }, - "node_modules/stdout-update/node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, "node_modules/stdout-update/node_modules/emoji-regex": { "version": "10.6.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", @@ -7576,6 +7510,24 @@ "node": ">=8" } }, + "node_modules/string-width-cjs/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -7632,19 +7584,20 @@ } }, "node_modules/tar": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.2.tgz", - "integrity": "sha512-7NyxrTE4Anh8km8iEy7o0QYPs+0JKBTj5ZaqHg6B39erLg0qYXN3BijtShwbsNSvQ+LN75+KV+C4QR/f6Gwnpg==", - "license": "BlueOak-1.0.0", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "license": "ISC", "dependencies": { - "@isaacs/fs-minipass": "^4.0.0", - "chownr": "^3.0.0", - "minipass": "^7.1.2", - "minizlib": "^3.1.0", - "yallist": "^5.0.0" + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" }, "engines": { - "node": ">=18" + "node": ">=10" } }, "node_modules/tar-fs": { @@ -7659,6 +7612,12 @@ "tar-stream": "^2.1.4" } }, + "node_modules/tar-fs/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, "node_modules/tar-stream": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", @@ -7675,13 +7634,13 @@ "node": ">=6" } }, - "node_modules/tar/node_modules/chownr": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", - "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", - "license": "BlueOak-1.0.0", + "node_modules/tar/node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "license": "ISC", "engines": { - "node": ">=18" + "node": ">=8" } }, "node_modules/tinybench": { @@ -7796,31 +7755,6 @@ "node": ">= 0.6" } }, - "node_modules/type-is/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/type-is/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -7836,9 +7770,9 @@ } }, "node_modules/undici": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.16.0.tgz", - "integrity": "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==", + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.18.2.tgz", + "integrity": "sha512-y+8YjDFzWdQlSE9N5nzKMT3g4a5UBX1HKowfdXh0uvAnTaqqwqB92Jt4UXBAeKekDs5IaDKyJFR4X1gYVCgXcw==", "license": "MIT", "engines": { "node": ">=20.18.1" @@ -7986,19 +7920,19 @@ } }, "node_modules/vitest": { - "version": "4.0.16", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.0.16.tgz", - "integrity": "sha512-E4t7DJ9pESL6E3I8nFjPa4xGUd3PmiWDLsDztS2qXSJWfHtbQnwAWylaBvSNY48I3vr8PTqIZlyK8TE3V3CA4Q==", + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.0.17.tgz", + "integrity": "sha512-FQMeF0DJdWY0iOnbv466n/0BudNdKj1l5jYgl5JVTwjSsZSlqyXFt/9+1sEyhR6CLowbZpV7O1sCHrzBhucKKg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.0.16", - "@vitest/mocker": "4.0.16", - "@vitest/pretty-format": "4.0.16", - "@vitest/runner": "4.0.16", - "@vitest/snapshot": "4.0.16", - "@vitest/spy": "4.0.16", - "@vitest/utils": "4.0.16", + "@vitest/expect": "4.0.17", + "@vitest/mocker": "4.0.17", + "@vitest/pretty-format": "4.0.17", + "@vitest/runner": "4.0.17", + "@vitest/snapshot": "4.0.17", + "@vitest/spy": "4.0.17", + "@vitest/utils": "4.0.17", "es-module-lexer": "^1.7.0", "expect-type": "^1.2.2", "magic-string": "^0.30.21", @@ -8026,10 +7960,10 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.0.16", - "@vitest/browser-preview": "4.0.16", - "@vitest/browser-webdriverio": "4.0.16", - "@vitest/ui": "4.0.16", + "@vitest/browser-playwright": "4.0.17", + "@vitest/browser-preview": "4.0.17", + "@vitest/browser-webdriverio": "4.0.17", + "@vitest/ui": "4.0.17", "happy-dom": "*", "jsdom": "*" }, @@ -8076,6 +8010,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation", "license": "MIT", "dependencies": { "iconv-lite": "0.6.3" @@ -8172,6 +8107,36 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -8188,13 +8153,10 @@ } }, "node_modules/yallist": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", - "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=18" - } + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" }, "node_modules/yargs": { "version": "17.7.2", @@ -8248,18 +8210,18 @@ } }, "node_modules/zod": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.2.1.tgz", - "integrity": "sha512-0wZ1IRqGGhMP76gLqz8EyfBXKk0J2qo2+H3fi4mcUP/KtTocoX08nmIAHl1Z2kJIZbZee8KOpBCSNPRgauucjw==", + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.5.tgz", + "integrity": "sha512-k7Nwx6vuWx1IJ9Bjuf4Zt1PEllcwe7cls3VNzm4CQ1/hgtFUK2bRNG3rvnpPUhFjmqJKAKtjV576KnUkHocg/g==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" } }, "node_modules/zod-to-json-schema": { - "version": "3.25.0", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.0.tgz", - "integrity": "sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==", + "version": "3.25.1", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.1.tgz", + "integrity": "sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==", "license": "ISC", "peerDependencies": { "zod": "^3.25 || ^4" diff --git a/package.json b/package.json index b1a641e..2cbe36e 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,6 @@ "cheerio": "^1.0.0-rc.12", "commander": "^12.0.0", "dotenv": "^16.3.1", - "faiss-node": "^0.5.1", "glob": "^10.3.10", "gray-matter": "^4.0.3", "inquirer": "^9.2.12", @@ -73,5 +72,8 @@ "@vitest/coverage-v8": "^4.0.16", "typescript": "^5.9.3", "vitest": "^4.0.16" + }, + "optionalDependencies": { + "faiss-node": "^0.5.1" } } diff --git a/src/cli.ts b/src/cli.ts index a44848a..c27127c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -94,6 +94,12 @@ program .option('--gpu-layers ', 'Number of GPU layers to offload (default: auto)', parseInt) .option('--context-size ', 'Context window size for local models (default: 32768)', parseInt) .option('--threads ', 'CPU threads for local inference (default: auto)', parseInt) + // Contextual retrieval options + .option('--contextual', 'Enable contextual retrieval for better chunk understanding (uses Claude API)') + .option('--contextual-local', 'Use local LLM for contextual retrieval instead of Claude API') + .option('--contextual-model ', 'Claude model for contextual retrieval (default: claude-3-haiku-20240307)') + .option('--contextual-ollama-model ', 'Ollama model for local contextual retrieval (default: qwen2.5-coder:7b)') + .option('--contextual-preview [n]', 'Preview contextual enrichment on N sample chunks (default: 10)', parseInt) .action(async (options) => { try { const configManager = new ConfigManager(); @@ -304,9 +310,126 @@ program ollamaHost: options.ollamaHost, gpuLayers: options.gpuLayers, contextSize: options.contextSize, - threads: options.threads + threads: options.threads, + // Contextual retrieval options + useContextualRetrieval: options.contextual || options.contextualLocal, + contextualLocal: options.contextualLocal, + contextualUseOllama: options.contextualLocal && options.useOllama, // Use Ollama if both flags set + contextualModel: options.contextualModel, + contextualOllamaHost: options.ollamaHost, // Reuse Ollama host if set + contextualLocalModel: options.contextualOllamaModel }; + // Handle --contextual-preview mode + if (options.contextualPreview !== undefined) { + spinner.stop(); + console.log(chalk.cyan.bold('\nšŸ” Contextual Retrieval Preview\n')); + + const sampleSize = typeof options.contextualPreview === 'number' ? options.contextualPreview : 10; + + // Import required modules + const { ASTChunker } = await import('./ast-chunker.js'); + const { ContextualRetrieval } = await import('./rag/contextual-retrieval.js'); + const simpleGit = await import('simple-git'); + const { glob } = await import('glob'); + const fsModule = await import('fs'); + + // Clone/prepare repo + const repoDir = options.repo.startsWith('http') || options.repo.includes('@') + ? `/tmp/semanticwiki-preview-${Date.now()}` + : path.resolve(options.repo); + + if (options.repo.startsWith('http') || options.repo.includes('@')) { + console.log(chalk.gray('Cloning repository...')); + const git = simpleGit.simpleGit(); + await git.clone(options.repo, repoDir, ['--depth', '1']); + } + + // Find source files + console.log(chalk.gray('Finding source files...')); + const searchPath = options.path ? path.join(repoDir, options.path) : repoDir; + const files = await glob('**/*.{ts,tsx,js,jsx,py,java,go,rs,rb,php,cs,cpp,c,h}', { + cwd: searchPath, + ignore: ['**/node_modules/**', '**/dist/**', '**/build/**', '**/.git/**', '**/vendor/**'], + }); + + // Chunk files + console.log(chalk.gray('Chunking files...')); + const chunker = new ASTChunker(); + const chunks: import('./ast-chunker.js').ASTChunk[] = []; + + for (const file of files.slice(0, 50)) { // Limit files for preview + try { + const relativePath = options.path ? path.join(options.path, file) : file; + const fileChunks = await chunker.chunkFile(relativePath, repoDir); + chunks.push(...fileChunks); + } catch { + // Skip files that fail to chunk + } + } + console.log(chalk.gray(` Found ${chunks.length} chunks from ${Math.min(files.length, 50)} files\n`)); + + // Initialize contextual retrieval + const contextualRetrieval = new ContextualRetrieval({ + enabled: true, + useLocal: options.contextualLocal || options.fullLocal, + useOllama: options.useOllama, + apiKey: config.apiKey, + model: options.contextualModel, + ollamaHost: options.ollamaHost, + localModel: options.contextualOllamaModel, + }); + + console.log(chalk.gray('Initializing LLM...')); + await contextualRetrieval.initialize(); + + // Run preview + console.log(chalk.gray(`Generating context for ${sampleSize} sample chunks...\n`)); + const preview = await contextualRetrieval.previewEnrichment(chunks, repoDir, sampleSize); + + // Display results + console.log(chalk.white.bold('Mode:'), chalk.yellow(preview.mode)); + console.log(chalk.white.bold('Total chunks:'), chalk.yellow(preview.totalChunks.toString())); + console.log(chalk.white.bold('Sample size:'), chalk.yellow(preview.sampleSize.toString())); + console.log(); + + // Count by status + const statusCounts = { success: 0, empty: 0, fallback: 0, error: 0 }; + for (const sample of preview.samples) { + statusCounts[sample.status]++; + } + + console.log(chalk.green(` āœ“ ${statusCounts.success} successful`)); + if (statusCounts.empty > 0) console.log(chalk.yellow(` āš  ${statusCounts.empty} empty responses`)); + if (statusCounts.error > 0) console.log(chalk.red(` āœ— ${statusCounts.error} errors`)); + console.log(); + + // Show samples + console.log(chalk.cyan.bold('Sample Results:\n')); + for (let i = 0; i < preview.samples.length; i++) { + const sample = preview.samples[i]; + const statusIcon = sample.status === 'success' ? 'āœ“' : sample.status === 'empty' ? 'āš ' : 'āœ—'; + const statusColor = sample.status === 'success' ? chalk.green : sample.status === 'empty' ? chalk.yellow : chalk.red; + + console.log(statusColor(`[${i + 1}] ${statusIcon} ${sample.filePath}`)); + if (sample.name) { + console.log(chalk.gray(` ${sample.chunkType || 'chunk'}: ${sample.name}`)); + } + console.log(chalk.white(' Context:'), chalk.cyan(sample.contextualPrefix || '(empty)')); + console.log(); + } + + // Cleanup + await contextualRetrieval.cleanup(); + + // Clean up temp dir if we cloned + if (options.repo.startsWith('http') || options.repo.includes('@')) { + fsModule.rmSync(repoDir, { recursive: true, force: true }); + } + + return; + } + // Choose generator based on options let generator; if (options.fullLocal) { @@ -1432,9 +1555,9 @@ program // Package command - create portable wiki package program .command('pack') - .description('Create a portable .archiwiki package from wiki directory') + .description('Create a portable .semantics package from wiki directory') .option('-o, --output ', 'Wiki directory to package', './wiki') - .option('-f, --file ', 'Output package file path (default: wiki name + .archiwiki)') + .option('-f, --file ', 'Output package file path (default: wiki name + .semantics)') .option('-n, --name ', 'Package name') .option('-d, --description ', 'Package description') .option('--source-repo ', 'Source repository URL') @@ -1448,11 +1571,11 @@ program process.exit(1); } - console.log(chalk.cyan.bold('\nšŸ“¦ Creating ArchiWiki Package\n')); + console.log(chalk.cyan.bold('\nšŸ“¦ Creating SemanticWiki Package\n')); const { createPackage } = await import('./package-format.js'); - const outputPath = options.file || path.join(path.dirname(wikiDir), `${path.basename(wikiDir)}.archiwiki`); + const outputPath = options.file || path.join(path.dirname(wikiDir), `${path.basename(wikiDir)}.semantics`); await createPackage({ wikiPath: wikiDir, @@ -1473,8 +1596,8 @@ program // Unpack command - extract portable wiki package program .command('unpack') - .description('Extract an .archiwiki package to a directory') - .argument('', 'Path to .archiwiki package file') + .description('Extract a .semantics package to a directory') + .argument('', 'Path to .semantics package file') .option('-o, --output ', 'Output directory', '.') .option('--wiki-only', 'Extract only wiki files, skip RAG index') .option('--info', 'Show package info without extracting') @@ -1524,7 +1647,7 @@ program return; } - console.log(chalk.cyan.bold('\nšŸ“¦ Extracting ArchiWiki Package\n')); + console.log(chalk.cyan.bold('\nšŸ“¦ Extracting SemanticWiki Package\n')); const manifest = await extractPackage({ packagePath: pkgPath, diff --git a/src/discovery/index.ts b/src/discovery/index.ts new file mode 100644 index 0000000..044ad89 --- /dev/null +++ b/src/discovery/index.ts @@ -0,0 +1,1465 @@ +/** + * Codebase Discovery Module + * + * Analyzes a codebase to discover its logical structure, domains, + * and relationships. Creates a hierarchical wiki plan that mirrors + * the actual architecture rather than just file types. + * + * Inspired by DeepWiki's approach of deep understanding before generation. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { glob } from 'glob'; + +/** + * Represents a discovered domain/module in the codebase + */ +export interface DiscoveredDomain { + /** Unique identifier for the domain */ + id: string; + /** Human-readable name */ + name: string; + /** Description of what this domain does */ + description: string; + /** Business purpose of this domain */ + businessPurpose?: string; + /** Files that belong to this domain */ + files: string[]; + /** Key components within this domain */ + components: DiscoveredComponent[]; + /** Related domains (by ID) */ + relatedDomains: string[]; + /** Suggested wiki page structure */ + suggestedPages: SuggestedPage[]; + /** Domain category for grouping */ + category: DomainCategory; + /** Confidence score (0-1) */ + confidence: number; +} + +/** + * Represents a component within a domain + */ +export interface DiscoveredComponent { + name: string; + type: ComponentType; + filePath: string; + description?: string; + /** For JCL jobs: the program/utility executed (IDCAMS, IEBGENER, etc.) */ + program?: string; + /** Brief function description extracted from comments or inferred */ + function?: string; + dependencies: string[]; + dependents: string[]; +} + +/** + * Component types + */ +export type ComponentType = + | 'entrypoint' + | 'controller' + | 'service' + | 'repository' + | 'model' + | 'utility' + | 'middleware' + | 'config' + | 'test' + | 'script' + | 'job' + | 'screen' + | 'copybook' + | 'unknown'; + +/** + * Domain categories for high-level grouping + */ +export type DomainCategory = + | 'core-application' + | 'data-layer' + | 'presentation' + | 'integration' + | 'batch-processing' + | 'infrastructure' + | 'documentation' + | 'testing' + | 'configuration'; + +/** + * Suggested wiki page + */ +export interface SuggestedPage { + slug: string; + title: string; + description: string; + sourcePaths: string[]; + pageType: 'overview' | 'feature' | 'reference' | 'guide' | 'relationship'; +} + +/** + * Relationship between components/domains + */ +export interface DiscoveredRelationship { + sourceId: string; + targetId: string; + type: RelationshipType; + description: string; +} + +export type RelationshipType = + | 'calls' + | 'imports' + | 'includes' + | 'references' + | 'data-flow' + | 'triggers' + | 'extends'; + +/** + * Complete discovery result + */ +export interface DiscoveryResult { + /** Project metadata */ + project: { + name: string; + type: ProjectType; + technologies: string[]; + description: string; + }; + /** Discovered domains/modules */ + domains: DiscoveredDomain[]; + /** Relationships between domains */ + relationships: DiscoveredRelationship[]; + /** Hierarchical wiki structure */ + wikiStructure: WikiStructure; + /** Statistics */ + stats: { + totalFiles: number; + totalDomains: number; + totalRelationships: number; + }; +} + +export type ProjectType = + | 'mainframe-cobol' + | 'web-application' + | 'api-service' + | 'cli-tool' + | 'library' + | 'monorepo' + | 'unknown'; + +/** + * Hierarchical wiki structure + */ +export interface WikiStructure { + sections: WikiSection[]; +} + +export interface WikiSection { + id: string; + title: string; + description: string; + pages: SuggestedPage[]; + subsections?: WikiSection[]; +} + +/** + * Discovery configuration + */ +export interface DiscoveryConfig { + /** Repository path */ + repoPath: string; + /** Target path within repo (optional) */ + targetPath?: string; + /** Verbose logging */ + verbose?: boolean; +} + +/** + * File pattern configurations for different project types + */ +const FILE_PATTERNS = { + cobol: { + programs: '**/*.{cbl,cob,CBL,COB}', + copybooks: '**/*.{cpy,CPY}', + jcl: '**/*.{jcl,JCL}', + bms: '**/*.{bms,BMS}', + ddl: '**/*.{ddl,DDL,sql,SQL}', + }, + web: { + components: '**/*.{tsx,jsx,vue,svelte}', + services: '**/services/**/*.{ts,js}', + controllers: '**/controllers/**/*.{ts,js}', + models: '**/models/**/*.{ts,js}', + config: '**/config/**/*.{ts,js,json}', + }, + general: { + source: '**/*.{ts,tsx,js,jsx,py,java,go,rs,rb,php,cs,cpp,c,h}', + config: '**/*.{json,yaml,yml,toml,ini}', + docs: '**/*.{md,rst,txt}', + scripts: '**/*.{sh,bash,ps1,bat}', + }, +}; + +/** + * Domain detection patterns + */ +const DOMAIN_PATTERNS: Array<{ + pattern: RegExp; + domain: string; + category: DomainCategory; + description: string; +}> = [ + // Authentication/Security + { + pattern: /auth|login|session|token|jwt|oauth|security|password|credential/i, + domain: 'authentication', + category: 'core-application', + description: 'User authentication and security', + }, + // User Management + { + pattern: /user|profile|account|member|customer/i, + domain: 'user-management', + category: 'core-application', + description: 'User account and profile management', + }, + // Transaction Processing + { + pattern: /transaction|payment|billing|invoice|order|checkout|cart/i, + domain: 'transaction-processing', + category: 'core-application', + description: 'Financial transactions and order processing', + }, + // Card/Account (Mainframe specific) + { + pattern: /card|acct|account|cust|customer/i, + domain: 'account-management', + category: 'core-application', + description: 'Account and card management', + }, + // Data Access + { + pattern: /repository|dao|database|db|query|storage|vsam|file/i, + domain: 'data-access', + category: 'data-layer', + description: 'Data storage and retrieval', + }, + // Batch Processing + { + pattern: /batch|job|scheduler|cron|task|queue/i, + domain: 'batch-processing', + category: 'batch-processing', + description: 'Batch jobs and scheduled tasks', + }, + // Screen/UI (Mainframe) + { + pattern: /screen|map|bms|panel|menu|display/i, + domain: 'screen-handling', + category: 'presentation', + description: 'User interface and screen handling', + }, + // API/Integration + { + pattern: /api|endpoint|route|controller|handler|service/i, + domain: 'api-services', + category: 'integration', + description: 'API endpoints and services', + }, + // Reports + { + pattern: /report|print|output|export/i, + domain: 'reporting', + category: 'core-application', + description: 'Reports and data export', + }, + // Configuration + { + pattern: /config|setting|env|parameter|option/i, + domain: 'configuration', + category: 'configuration', + description: 'System configuration', + }, + // Testing + { + pattern: /test|spec|mock|fixture/i, + domain: 'testing', + category: 'testing', + description: 'Testing and quality assurance', + }, +]; + +/** + * CodebaseDiscovery - Analyzes codebases to discover structure and domains + */ +export class CodebaseDiscovery { + private config: DiscoveryConfig; + private projectType: ProjectType = 'unknown'; + private fileIndex: Map = new Map(); // domain -> files + private componentIndex: Map = new Map(); + + constructor(config: DiscoveryConfig) { + this.config = config; + } + + private log(...args: any[]): void { + if (this.config.verbose) { + console.log('[Discovery]', ...args); + } + } + + /** + * Run full discovery on the codebase + */ + async discover(): Promise { + console.log('šŸ” Starting codebase discovery...\n'); + + // Phase 1: Detect project type + this.projectType = await this.detectProjectType(); + console.log(` Project type: ${this.projectType}`); + + // Phase 2: Scan and categorize files + const files = await this.scanFiles(); + console.log(` Files scanned: ${files.length}`); + + // Phase 3: Analyze file contents and detect domains + const domains = await this.detectDomains(files); + console.log(` Domains discovered: ${domains.length}`); + + // Phase 4: Discover relationships + const relationships = await this.discoverRelationships(domains); + console.log(` Relationships found: ${relationships.length}`); + + // Phase 5: Generate hierarchical wiki structure + const wikiStructure = this.generateWikiStructure(domains, relationships); + console.log(` Wiki sections: ${wikiStructure.sections.length}`); + + // Phase 6: Extract project metadata + const project = await this.extractProjectMetadata(); + + console.log('\nāœ… Discovery complete!\n'); + + return { + project, + domains, + relationships, + wikiStructure, + stats: { + totalFiles: files.length, + totalDomains: domains.length, + totalRelationships: relationships.length, + }, + }; + } + + /** + * Detect the project type based on files present + */ + private async detectProjectType(): Promise { + const basePath = this.config.targetPath + ? path.join(this.config.repoPath, this.config.targetPath) + : this.config.repoPath; + + // Check for COBOL/Mainframe indicators + const cobolFiles = await glob('**/*.{cbl,cob,CBL,COB}', { + cwd: basePath, + ignore: ['**/node_modules/**'], + }); + if (cobolFiles.length > 0) { + return 'mainframe-cobol'; + } + + // Check for web app indicators + const hasPackageJson = fs.existsSync(path.join(basePath, 'package.json')); + const hasReactOrVue = await glob('**/*.{tsx,jsx,vue}', { + cwd: basePath, + ignore: ['**/node_modules/**'], + }); + if (hasPackageJson && hasReactOrVue.length > 0) { + return 'web-application'; + } + + // Check for API service indicators + const hasRoutes = await glob('**/routes/**/*.{ts,js}', { + cwd: basePath, + ignore: ['**/node_modules/**'], + }); + const hasControllers = await glob('**/controllers/**/*.{ts,js}', { + cwd: basePath, + ignore: ['**/node_modules/**'], + }); + if (hasRoutes.length > 0 || hasControllers.length > 0) { + return 'api-service'; + } + + // Check for CLI tool indicators + const hasBin = fs.existsSync(path.join(basePath, 'bin')); + const hasCliInPackage = hasPackageJson && this.checkPackageJsonForCli(basePath); + if (hasBin || hasCliInPackage) { + return 'cli-tool'; + } + + return 'unknown'; + } + + private checkPackageJsonForCli(basePath: string): boolean { + try { + const pkgJson = JSON.parse(fs.readFileSync(path.join(basePath, 'package.json'), 'utf-8')); + return !!pkgJson.bin; + } catch { + return false; + } + } + + /** + * Scan all relevant files in the codebase + */ + private async scanFiles(): Promise { + const basePath = this.config.targetPath + ? path.join(this.config.repoPath, this.config.targetPath) + : this.config.repoPath; + + let patterns: string[]; + + switch (this.projectType) { + case 'mainframe-cobol': + patterns = [ + FILE_PATTERNS.cobol.programs, + FILE_PATTERNS.cobol.copybooks, + FILE_PATTERNS.cobol.jcl, + FILE_PATTERNS.cobol.bms, + FILE_PATTERNS.cobol.ddl, + FILE_PATTERNS.general.docs, + FILE_PATTERNS.general.scripts, + ]; + break; + default: + patterns = [ + FILE_PATTERNS.general.source, + FILE_PATTERNS.general.config, + FILE_PATTERNS.general.docs, + FILE_PATTERNS.general.scripts, + ]; + } + + const allFiles: string[] = []; + for (const pattern of patterns) { + const files = await glob(pattern, { + cwd: basePath, + ignore: [ + '**/node_modules/**', + '**/dist/**', + '**/build/**', + '**/.git/**', + '**/vendor/**', + '**/__pycache__/**', + ], + }); + allFiles.push(...files); + } + + return [...new Set(allFiles)]; // Deduplicate + } + + /** + * Detect domains by analyzing file paths and contents + */ + private async detectDomains(files: string[]): Promise { + const basePath = this.config.targetPath + ? path.join(this.config.repoPath, this.config.targetPath) + : this.config.repoPath; + + // Group files by detected domain + const domainFiles = new Map(); + const domainMeta = new Map(); + + for (const file of files) { + const matchedDomain = this.matchFileToDomain(file, basePath); + if (matchedDomain) { + const existing = domainFiles.get(matchedDomain.domain) || []; + existing.push(file); + domainFiles.set(matchedDomain.domain, existing); + domainMeta.set(matchedDomain.domain, { + category: matchedDomain.category, + description: matchedDomain.description, + }); + } + } + + // Also group by directory structure for mainframe projects + if (this.projectType === 'mainframe-cobol') { + this.groupByDirectoryStructure(files, domainFiles, domainMeta); + } + + // Convert to DiscoveredDomain objects + const domains: DiscoveredDomain[] = []; + for (const [domainId, domainFileList] of domainFiles) { + const meta = domainMeta.get(domainId) || { + category: 'core-application' as DomainCategory, + description: `${domainId} domain`, + }; + + const components = await this.extractComponents(domainFileList, basePath); + const suggestedPages = this.generateSuggestedPages(domainId, domainFileList, components); + + domains.push({ + id: domainId, + name: this.formatDomainName(domainId), + description: meta.description, + businessPurpose: this.inferBusinessPurpose(domainId, domainFileList), + files: domainFileList, + components, + relatedDomains: [], + suggestedPages, + category: meta.category, + confidence: 0.8, + }); + } + + // Find related domains + this.linkRelatedDomains(domains); + + return domains.sort((a, b) => { + // Sort by category priority then by name + const categoryOrder: DomainCategory[] = [ + 'core-application', + 'presentation', + 'data-layer', + 'batch-processing', + 'integration', + 'configuration', + 'infrastructure', + 'documentation', + 'testing', + ]; + const aIdx = categoryOrder.indexOf(a.category); + const bIdx = categoryOrder.indexOf(b.category); + if (aIdx !== bIdx) return aIdx - bIdx; + return a.name.localeCompare(b.name); + }); + } + + /** + * Match a file to a domain based on patterns + */ + private matchFileToDomain( + file: string, + basePath: string + ): { domain: string; category: DomainCategory; description: string } | null { + const fullPath = path.join(basePath, file); + const fileName = path.basename(file); + const dirName = path.dirname(file); + + // Try content-based matching for source files + const ext = path.extname(file).toLowerCase(); + const sourceExts = ['.ts', '.js', '.tsx', '.jsx', '.py', '.java', '.cbl', '.cob']; + + let contentSample = ''; + if (sourceExts.includes(ext)) { + try { + contentSample = fs.readFileSync(fullPath, 'utf-8').slice(0, 2000); + } catch { + // Ignore read errors + } + } + + // Match against domain patterns + for (const pattern of DOMAIN_PATTERNS) { + if ( + pattern.pattern.test(fileName) || + pattern.pattern.test(dirName) || + pattern.pattern.test(contentSample) + ) { + return { + domain: pattern.domain, + category: pattern.category, + description: pattern.description, + }; + } + } + + // Fallback: categorize by file type + if (ext === '.cbl' || ext === '.cob') { + return { + domain: 'cobol-programs', + category: 'core-application', + description: 'COBOL application programs', + }; + } + if (ext === '.cpy') { + return { + domain: 'copybooks', + category: 'data-layer', + description: 'COBOL copybook definitions', + }; + } + if (ext === '.jcl') { + return { + domain: 'jcl-jobs', + category: 'batch-processing', + description: 'JCL batch job definitions', + }; + } + if (ext === '.bms') { + return { + domain: 'screen-maps', + category: 'presentation', + description: 'BMS screen map definitions', + }; + } + if (ext === '.ddl' || ext === '.sql') { + return { + domain: 'database-schema', + category: 'data-layer', + description: 'Database schema definitions', + }; + } + if (ext === '.md') { + return { + domain: 'documentation', + category: 'documentation', + description: 'Project documentation', + }; + } + if (ext === '.sh' || ext === '.bash') { + return { + domain: 'scripts', + category: 'infrastructure', + description: 'Shell scripts and utilities', + }; + } + + return null; + } + + /** + * Group files by directory structure (for mainframe projects) + */ + private groupByDirectoryStructure( + files: string[], + domainFiles: Map, + domainMeta: Map + ): void { + // Find top-level directories + const topDirs = new Set(); + for (const file of files) { + const parts = file.split('/'); + if (parts.length > 1) { + topDirs.add(parts[0]); + } + } + + // Create domain for each significant directory + for (const dir of topDirs) { + const dirFiles = files.filter((f) => f.startsWith(dir + '/')); + if (dirFiles.length >= 3) { + // Only create domain for dirs with multiple files + const domainId = `${dir}-module`; + if (!domainFiles.has(domainId)) { + domainFiles.set(domainId, dirFiles); + domainMeta.set(domainId, { + category: this.inferCategoryFromDir(dir), + description: `${this.formatDomainName(dir)} module`, + }); + } + } + } + } + + private inferCategoryFromDir(dir: string): DomainCategory { + const lower = dir.toLowerCase(); + if (lower.includes('app') || lower.includes('src')) return 'core-application'; + if (lower.includes('sample') || lower.includes('example')) return 'documentation'; + if (lower.includes('test')) return 'testing'; + if (lower.includes('script') || lower.includes('tool')) return 'infrastructure'; + if (lower.includes('doc')) return 'documentation'; + return 'core-application'; + } + + /** + * Check if a file is documentation/noise that should be excluded from component listings + */ + private isDocumentationFile(file: string): boolean { + const fileName = path.basename(file).toLowerCase(); + const ext = path.extname(file).toLowerCase(); + + // Documentation file extensions + const docExtensions = ['.md', '.txt', '.rst', '.adoc', '.pdf', '.doc', '.docx']; + if (docExtensions.includes(ext)) { + return true; + } + + // Common documentation file names + const docFileNames = [ + 'readme', 'license', 'changelog', 'contributing', 'authors', + 'copying', 'todo', 'history', 'news', 'thanks', 'credits', + 'security', 'code_of_conduct', 'codeowners', 'makefile' + ]; + const baseNameLower = fileName.replace(ext, ''); + if (docFileNames.includes(baseNameLower)) { + return true; + } + + // Hidden config files + if (fileName.startsWith('.')) { + return true; + } + + return false; + } + + /** + * Extract components from files + */ + private async extractComponents( + files: string[], + basePath: string + ): Promise { + const components: DiscoveredComponent[] = []; + + // Filter out documentation/noise files - only include actual source code + const sourceFiles = files.filter(f => !this.isDocumentationFile(f)); + + for (const file of sourceFiles.slice(0, 100)) { + // Increased limit for better coverage + const fullPath = path.join(basePath, file); + const ext = path.extname(file).toLowerCase(); + const fileName = path.basename(file, ext); + + let type: ComponentType = 'unknown'; + let program: string | undefined; + let functionDesc: string | undefined; + + // Detect component type + if (file.includes('controller') || file.includes('handler')) { + type = 'controller'; + } else if (file.includes('service')) { + type = 'service'; + } else if (file.includes('repository') || file.includes('dao')) { + type = 'repository'; + } else if (file.includes('model') || file.includes('entity')) { + type = 'model'; + } else if (file.includes('util') || file.includes('helper')) { + type = 'utility'; + } else if (file.includes('middleware')) { + type = 'middleware'; + } else if (file.includes('config')) { + type = 'config'; + } else if (file.includes('test') || file.includes('spec')) { + type = 'test'; + } else if (ext === '.jcl') { + type = 'job'; + // Extract program and function from JCL content + const jclInfo = this.extractJclInfo(fullPath, fileName); + program = jclInfo.program; + functionDesc = jclInfo.function; + } else if (ext === '.bms') { + type = 'screen'; + functionDesc = this.inferBmsFunction(fileName); + } else if (ext === '.cpy') { + type = 'copybook'; + functionDesc = this.inferCopybookFunction(fileName); + } else if (ext === '.sh' || ext === '.bash') { + type = 'script'; + } else if (ext === '.cbl' || ext === '.cob') { + type = 'entrypoint'; + functionDesc = this.extractCobolFunction(fullPath, fileName); + } + + components.push({ + name: fileName, + type, + filePath: file, + description: functionDesc, + program, + function: functionDesc, + dependencies: [], + dependents: [], + }); + } + + return components; + } + + /** + * Extract program and function from JCL file content + */ + private extractJclInfo(fullPath: string, fileName: string): { program?: string; function?: string } { + try { + const content = fs.readFileSync(fullPath, 'utf-8').toUpperCase(); + + // Extract program from EXEC PGM= or EXEC statement + let program: string | undefined; + const pgmMatch = content.match(/EXEC\s+PGM=(\w+)/); + if (pgmMatch) { + program = pgmMatch[1]; + } else { + // Check for common utilities + if (content.includes('IDCAMS')) program = 'IDCAMS'; + else if (content.includes('IEBGENER')) program = 'IEBGENER'; + else if (content.includes('IEFBR14')) program = 'IEFBR14'; + else if (content.includes('SORT')) program = 'SORT'; + else if (content.includes('IKJEFT01')) program = 'IKJEFT01'; + else if (content.includes('DFSRRC00')) program = 'DFSRRC00'; + } + + // Infer function from job name patterns and content + let functionDesc = this.inferJclFunction(fileName, content); + + return { program, function: functionDesc }; + } catch { + return { function: this.inferJclFunction(fileName, '') }; + } + } + + /** + * Infer JCL job function from name and content patterns + */ + private inferJclFunction(jobName: string, content: string): string { + const name = jobName.toUpperCase(); + + // Common JCL job name patterns + const patterns: Array<{ pattern: RegExp; function: string }> = [ + { pattern: /USRSEC|SECURITY|SEC/i, function: 'Load/Manage User Security' }, + { pattern: /DEFGDG|GDG/i, function: 'Setup GDG Bases' }, + { pattern: /ACCTFILE|ACCT/i, function: 'Refresh Account Master' }, + { pattern: /CARDFILE|CARD(?!DEMO)/i, function: 'Refresh Card Master' }, + { pattern: /CUSTFILE|CUST/i, function: 'Refresh Customer Master' }, + { pattern: /DISCGRP|DISCL/i, function: 'Load Disclosure Group File' }, + { pattern: /TRANFILE|TRNFILE/i, function: 'Load Transaction Master' }, + { pattern: /TRANCATG|TRANCAT/i, function: 'Load Transaction Category Types' }, + { pattern: /TRANTYPE|TRNTYPE/i, function: 'Load Transaction Type File' }, + { pattern: /XREFFILE|XREF/i, function: 'Account/Card/Customer Cross Reference' }, + { pattern: /CLOSEFIL|CLOSE/i, function: 'Close VSAM Files in CICS' }, + { pattern: /TCATBALF?|CATBAL/i, function: 'Refresh Transaction Category Balance' }, + { pattern: /LISTCAT/i, function: 'List VSAM Catalog Entries' }, + { pattern: /OPENFIL|OPEN/i, function: 'Open VSAM Files in CICS' }, + { pattern: /READCARD/i, function: 'Read Card Master File' }, + { pattern: /BACKUP|BKP/i, function: 'Backup Data Files' }, + { pattern: /RESTORE|RST/i, function: 'Restore Data Files' }, + { pattern: /LOAD/i, function: 'Initial Data Load' }, + { pattern: /DAILY|DLY/i, function: 'Daily Processing' }, + { pattern: /WEEKLY|WKY/i, function: 'Weekly Processing' }, + { pattern: /MONTHLY|MTH/i, function: 'Monthly Processing' }, + { pattern: /REPORT|RPT/i, function: 'Generate Reports' }, + { pattern: /DELETE|DEL/i, function: 'Delete/Cleanup Records' }, + { pattern: /CREATE|CRT/i, function: 'Create Data Structures' }, + ]; + + for (const { pattern, function: func } of patterns) { + if (pattern.test(name)) { + return func; + } + } + + // Check content for clues + if (content.includes('DEFINE CLUSTER')) return 'Define VSAM Cluster'; + if (content.includes('REPRO')) return 'Copy/Refresh Data'; + if (content.includes('DELETE')) return 'Delete Data/Files'; + if (content.includes('PRINT')) return 'Print/List Data'; + + return 'Batch Job Processing'; + } + + /** + * Infer BMS screen function from name + */ + private inferBmsFunction(fileName: string): string { + const name = fileName.toUpperCase(); + + const patterns: Array<{ pattern: RegExp; function: string }> = [ + { pattern: /MENU/i, function: 'Main Menu Screen' }, + { pattern: /SIGNON|LOGIN|LOGON/i, function: 'Sign-on/Login Screen' }, + { pattern: /ACCT/i, function: 'Account Display/Entry Screen' }, + { pattern: /CARD/i, function: 'Card Information Screen' }, + { pattern: /CUST/i, function: 'Customer Information Screen' }, + { pattern: /TRAN/i, function: 'Transaction Entry/Display Screen' }, + { pattern: /REPORT|RPT/i, function: 'Report Display Screen' }, + { pattern: /ADMIN/i, function: 'Administration Screen' }, + { pattern: /HELP/i, function: 'Help Screen' }, + { pattern: /ERROR|ERR/i, function: 'Error Display Screen' }, + ]; + + for (const { pattern, function: func } of patterns) { + if (pattern.test(name)) { + return func; + } + } + + return 'CICS Screen Map'; + } + + /** + * Infer copybook function from name + */ + private inferCopybookFunction(fileName: string): string { + const name = fileName.toUpperCase(); + + const patterns: Array<{ pattern: RegExp; function: string }> = [ + { pattern: /ACCT/i, function: 'Account Record Layout' }, + { pattern: /CARD/i, function: 'Card Record Layout' }, + { pattern: /CUST/i, function: 'Customer Record Layout' }, + { pattern: /TRAN/i, function: 'Transaction Record Layout' }, + { pattern: /USER/i, function: 'User Record Layout' }, + { pattern: /XREF/i, function: 'Cross-Reference Layout' }, + { pattern: /COMM|COMMON/i, function: 'Common Data Structures' }, + { pattern: /DATE/i, function: 'Date Handling Structures' }, + { pattern: /ERROR|ERR/i, function: 'Error Handling Structures' }, + ]; + + for (const { pattern, function: func } of patterns) { + if (pattern.test(name)) { + return func; + } + } + + return 'Data Structure Definition'; + } + + /** + * Extract COBOL program function from content + */ + private extractCobolFunction(fullPath: string, fileName: string): string { + try { + const content = fs.readFileSync(fullPath, 'utf-8'); + const lines = content.split('\n').slice(0, 50); // Check first 50 lines + + // Look for PROGRAM-ID comment or description + for (const line of lines) { + // Check for descriptive comments + if (line.match(/^\s*\*.*(?:PROGRAM|MODULE|ROUTINE).*(?:FOR|TO|:)/i)) { + const desc = line.replace(/^\s*\*+\s*/, '').trim(); + if (desc.length > 10 && desc.length < 100) { + return desc; + } + } + } + + // Infer from program name patterns + return this.inferCobolFunction(fileName); + } catch { + return this.inferCobolFunction(fileName); + } + } + + /** + * Infer COBOL program function from name + */ + private inferCobolFunction(fileName: string): string { + const name = fileName.toUpperCase(); + + const patterns: Array<{ pattern: RegExp; function: string }> = [ + { pattern: /MENU/i, function: 'Main Menu Handler' }, + { pattern: /SIGNON|LOGIN|LOGON/i, function: 'User Sign-on Processing' }, + { pattern: /ACCTVIEW|ACTVW/i, function: 'Account View/Display' }, + { pattern: /ACCTADD|ACTADD/i, function: 'Account Add/Create' }, + { pattern: /ACCTUPD|ACTUPD/i, function: 'Account Update' }, + { pattern: /CARDVIEW|CRDVW/i, function: 'Card View/Display' }, + { pattern: /CARDADD|CRDADD/i, function: 'Card Add/Create' }, + { pattern: /TRANVIEW|TRNVW/i, function: 'Transaction View' }, + { pattern: /TRANADD|TRNADD/i, function: 'Transaction Add' }, + { pattern: /REPORT|RPT/i, function: 'Report Generation' }, + { pattern: /ADMIN/i, function: 'Administration Functions' }, + { pattern: /BATCH|BAT/i, function: 'Batch Processing' }, + { pattern: /UTIL/i, function: 'Utility Functions' }, + ]; + + for (const { pattern, function: func } of patterns) { + if (pattern.test(name)) { + return func; + } + } + + return 'COBOL Program'; + } + + /** + * Generate suggested wiki pages for a domain + */ + private generateSuggestedPages( + domainId: string, + files: string[], + components: DiscoveredComponent[] + ): SuggestedPage[] { + const pages: SuggestedPage[] = []; + const domainName = this.formatDomainName(domainId); + + // Always create an overview page + pages.push({ + slug: `${domainId}.md`, + title: domainName, + description: `Overview of the ${domainName} domain`, + sourcePaths: files.slice(0, 10), + pageType: 'overview', + }); + + // Create feature pages for significant components + const significantComponents = components.filter((c) => + ['controller', 'service', 'entrypoint', 'job'].includes(c.type) + ); + + for (const comp of significantComponents.slice(0, 5)) { + pages.push({ + slug: `${domainId}-${comp.name.toLowerCase()}.md`, + title: `${domainName}: ${comp.name}`, + description: `Detailed documentation for ${comp.name}`, + sourcePaths: [comp.filePath], + pageType: 'feature', + }); + } + + return pages; + } + + /** + * Format domain ID into human-readable name + */ + private formatDomainName(domainId: string): string { + return domainId + .split('-') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + } + + /** + * Infer business purpose from domain and files + */ + private inferBusinessPurpose(domainId: string, files: string[]): string { + const purposes: Record = { + authentication: 'Secure user access and session management', + 'user-management': 'Manage user accounts, profiles, and preferences', + 'transaction-processing': 'Handle financial transactions and order processing', + 'account-management': 'Manage customer accounts and card information', + 'data-access': 'Provide reliable data storage and retrieval', + 'batch-processing': 'Execute scheduled background tasks and reports', + 'screen-handling': 'Present user interface and capture input', + 'api-services': 'Expose functionality via API endpoints', + reporting: 'Generate business reports and analytics', + configuration: 'Manage system settings and parameters', + 'cobol-programs': 'Core business logic and processing', + copybooks: 'Shared data structure definitions', + 'jcl-jobs': 'Batch job execution and scheduling', + 'screen-maps': 'Terminal screen layouts and field definitions', + 'database-schema': 'Database table and relationship definitions', + }; + + return purposes[domainId] || `Supports ${this.formatDomainName(domainId)} functionality`; + } + + /** + * Link related domains based on file proximity and naming + */ + private linkRelatedDomains(domains: DiscoveredDomain[]): void { + for (const domain of domains) { + const related: string[] = []; + + for (const other of domains) { + if (other.id === domain.id) continue; + + // Check for file path overlap + const overlap = domain.files.some((f1) => + other.files.some((f2) => { + const dir1 = path.dirname(f1); + const dir2 = path.dirname(f2); + return dir1 === dir2 || dir1.startsWith(dir2) || dir2.startsWith(dir1); + }) + ); + + // Check for naming similarity + const nameSimilarity = + domain.id.includes(other.id.split('-')[0]) || other.id.includes(domain.id.split('-')[0]); + + if (overlap || nameSimilarity) { + related.push(other.id); + } + } + + domain.relatedDomains = related.slice(0, 5); + } + } + + /** + * Discover relationships between domains + */ + private async discoverRelationships( + domains: DiscoveredDomain[] + ): Promise { + const relationships: DiscoveredRelationship[] = []; + + // Create relationships based on domain links + for (const domain of domains) { + for (const relatedId of domain.relatedDomains) { + relationships.push({ + sourceId: domain.id, + targetId: relatedId, + type: 'references', + description: `${domain.name} references ${this.formatDomainName(relatedId)}`, + }); + } + } + + // Add data flow relationships + const dataLayer = domains.find((d) => d.category === 'data-layer'); + const coreApps = domains.filter((d) => d.category === 'core-application'); + + if (dataLayer) { + for (const app of coreApps) { + relationships.push({ + sourceId: app.id, + targetId: dataLayer.id, + type: 'data-flow', + description: `${app.name} reads/writes data via ${dataLayer.name}`, + }); + } + } + + return relationships; + } + + /** + * Generate hierarchical wiki structure + */ + private generateWikiStructure( + domains: DiscoveredDomain[], + relationships: DiscoveredRelationship[] + ): WikiStructure { + const sections: WikiSection[] = []; + + // Group domains by category + const byCategory = new Map(); + for (const domain of domains) { + const existing = byCategory.get(domain.category) || []; + existing.push(domain); + byCategory.set(domain.category, existing); + } + + // Create section for each category with domains + const categoryMeta: Record = { + 'core-application': { + title: 'Core Application', + description: 'Main application logic and business processes', + }, + presentation: { + title: 'User Interface', + description: 'Screen handling and user interaction', + }, + 'data-layer': { + title: 'Data Layer', + description: 'Data storage, schemas, and access patterns', + }, + 'batch-processing': { + title: 'Batch Processing', + description: 'Scheduled jobs and background processing', + }, + integration: { + title: 'Integration', + description: 'External services and API integrations', + }, + configuration: { + title: 'Configuration', + description: 'System configuration and settings', + }, + infrastructure: { + title: 'Infrastructure', + description: 'Deployment scripts and utilities', + }, + documentation: { + title: 'Documentation', + description: 'Project documentation and guides', + }, + testing: { + title: 'Testing', + description: 'Test suites and quality assurance', + }, + }; + + for (const [category, categoryDomains] of byCategory) { + if (categoryDomains.length === 0) continue; + + const meta = categoryMeta[category]; + const sectionPages: SuggestedPage[] = []; + + // Add overview page for the section + sectionPages.push({ + slug: `${category}.md`, + title: meta.title, + description: meta.description, + sourcePaths: categoryDomains.flatMap((d) => d.files.slice(0, 5)), + pageType: 'overview', + }); + + // Add pages from each domain + for (const domain of categoryDomains) { + sectionPages.push(...domain.suggestedPages); + } + + sections.push({ + id: category, + title: meta.title, + description: meta.description, + pages: sectionPages, + }); + } + + // Add relationship section if we have cross-cutting concerns + if (relationships.length > 5) { + sections.push({ + id: 'relationships', + title: 'System Relationships', + description: 'How components interact and data flows through the system', + pages: [ + { + slug: 'data-flow.md', + title: 'Data Flow', + description: 'How data moves through the system', + sourcePaths: [], + pageType: 'relationship', + }, + { + slug: 'integration-points.md', + title: 'Integration Points', + description: 'How modules connect and communicate', + sourcePaths: [], + pageType: 'relationship', + }, + ], + }); + } + + return { sections }; + } + + /** + * Extract project metadata + */ + private async extractProjectMetadata(): Promise { + const basePath = this.config.repoPath; + + let name = path.basename(basePath); + let description = ''; + const technologies: string[] = []; + + // Try to read from package.json + try { + const pkgPath = path.join(basePath, 'package.json'); + if (fs.existsSync(pkgPath)) { + const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')); + name = pkg.name || name; + description = pkg.description || ''; + if (pkg.dependencies) { + technologies.push(...Object.keys(pkg.dependencies).slice(0, 10)); + } + } + } catch { + // Ignore + } + + // Try to read from README + try { + const readmePath = path.join(basePath, 'README.md'); + if (fs.existsSync(readmePath) && !description) { + const readme = fs.readFileSync(readmePath, 'utf-8'); + // Extract first paragraph as description + const firstPara = readme.split('\n\n')[0]?.replace(/^#.*\n/, '').trim(); + if (firstPara && firstPara.length < 500) { + description = firstPara; + } + } + } catch { + // Ignore + } + + // Add technology indicators + if (this.projectType === 'mainframe-cobol') { + technologies.push('COBOL', 'CICS', 'JCL', 'VSAM'); + } + + return { + name, + type: this.projectType, + technologies, + description: description || `${name} project`, + }; + } +} + +/** + * Generate a hierarchical index.md from discovery results + */ +export function generateHierarchicalIndex(result: DiscoveryResult): string { + const lines: string[] = []; + + // Header + lines.push(`# ${result.project.name}`); + lines.push(''); + lines.push(result.project.description); + lines.push(''); + + // Technology badges + if (result.project.technologies.length > 0) { + lines.push( + '**Technologies:** ' + result.project.technologies.slice(0, 8).join(' ā€¢ ') + ); + lines.push(''); + } + + lines.push('---'); + lines.push(''); + + // Navigation sections + for (const section of result.wikiStructure.sections) { + lines.push(`## ${section.title}`); + lines.push(''); + lines.push(`> ${section.description}`); + lines.push(''); + + // Pages in this section + for (const page of section.pages) { + const icon = getPageIcon(page.pageType); + lines.push(`- ${icon} [${page.title}](./${page.slug}) ā€“ ${page.description}`); + } + lines.push(''); + } + + // Quick reference table + lines.push('---'); + lines.push(''); + lines.push('## Quick Reference'); + lines.push(''); + lines.push('| Domain | Files | Category |'); + lines.push('|--------|-------|----------|'); + + for (const domain of result.domains.slice(0, 15)) { + lines.push( + `| [${domain.name}](./${domain.id}.md) | ${domain.files.length} | ${domain.category} |` + ); + } + + lines.push(''); + lines.push('---'); + lines.push(''); + lines.push('*Generated by SemanticWiki with hierarchical discovery*'); + + return lines.join('\n'); +} + +function getPageIcon(pageType: SuggestedPage['pageType']): string { + switch (pageType) { + case 'overview': + return 'šŸ“š'; + case 'feature': + return 'āš™ļø'; + case 'reference': + return 'šŸ“–'; + case 'guide': + return 'šŸ“'; + case 'relationship': + return 'šŸ”—'; + default: + return 'šŸ“„'; + } +} + +/** + * Generate a component summary table for a domain (DeepWiki style) + * Creates tables like: "Job | Program | Function" for batch jobs + */ +export function generateComponentTable( + domain: DiscoveredDomain, + options: { includeFilePath?: boolean } = {} +): string { + const lines: string[] = []; + const components = domain.components.filter(c => + c.type !== 'unknown' && c.function + ); + + if (components.length === 0) { + return ''; + } + + // Different table formats based on component types + const jobs = components.filter(c => c.type === 'job'); + const screens = components.filter(c => c.type === 'screen'); + const copybooks = components.filter(c => c.type === 'copybook'); + const programs = components.filter(c => c.type === 'entrypoint' || c.type === 'service'); + + // Batch Components table (JCL jobs) + if (jobs.length > 0) { + lines.push('### Batch Components'); + lines.push(''); + lines.push('| Job | Program | Function |'); + lines.push('|-----|---------|----------|'); + for (const job of jobs) { + const jobName = job.name.toUpperCase(); + const program = job.program || '-'; + const func = job.function || '-'; + lines.push(`| ${jobName} | ${program} | ${func} |`); + } + lines.push(''); + } + + // Screen Components table (BMS maps) + if (screens.length > 0) { + lines.push('### Screen Components'); + lines.push(''); + lines.push('| Screen | Function |'); + lines.push('|--------|----------|'); + for (const screen of screens) { + const screenName = screen.name.toUpperCase(); + const func = screen.function || 'Screen Map'; + lines.push(`| ${screenName} | ${func} |`); + } + lines.push(''); + } + + // Data Structures table (Copybooks) + if (copybooks.length > 0) { + lines.push('### Data Structures'); + lines.push(''); + lines.push('| Copybook | Purpose |'); + lines.push('|----------|---------|'); + for (const copybook of copybooks) { + const name = copybook.name.toUpperCase(); + const func = copybook.function || 'Data Definition'; + lines.push(`| ${name} | ${func} |`); + } + lines.push(''); + } + + // Program Components table + if (programs.length > 0) { + lines.push('### Program Components'); + lines.push(''); + lines.push('| Program | Function |'); + lines.push('|---------|----------|'); + for (const program of programs) { + const name = program.name.toUpperCase(); + const func = program.function || 'Application Logic'; + lines.push(`| ${name} | ${func} |`); + } + lines.push(''); + } + + return lines.join('\n'); +} + +/** + * Get all component tables for all domains in a discovery result + */ +export function generateAllComponentTables(result: DiscoveryResult): Map { + const tables = new Map(); + + for (const domain of result.domains) { + const table = generateComponentTable(domain); + if (table) { + tables.set(domain.id, table); + } + } + + return tables; +} + +export default CodebaseDiscovery; diff --git a/src/llm/index.ts b/src/llm/index.ts index 07ab189..1d62c13 100644 --- a/src/llm/index.ts +++ b/src/llm/index.ts @@ -61,6 +61,7 @@ export async function createLLMProvider(options: CreateProviderOptions = {}): Pr const family = options.modelFamily || 'gpt-oss'; const familyNames: Record = { 'gpt-oss': 'GPT-OSS (21B)', + 'completion': 'Qwen2.5-Coder (1.5B)', 'qwen': 'Qwen', 'lfm': 'LFM (LiquidAI)', }; diff --git a/src/llm/model-manager.ts b/src/llm/model-manager.ts index 0effcb5..3e72fc8 100644 --- a/src/llm/model-manager.ts +++ b/src/llm/model-manager.ts @@ -16,8 +16,10 @@ import type { HardwareProfile, ModelRecommendation, ProgressCallback } from './t /** * Model family type for filtering models + * - 'gpt-oss': Large agentic model with tool calling (for wiki generation) + * - 'completion': Small fast model for text completion tasks (for contextual retrieval) */ -export type ModelFamily = 'gpt-oss'; +export type ModelFamily = 'gpt-oss' | 'completion'; /** * Extended model recommendation with family @@ -30,7 +32,7 @@ interface ModelRegistryEntry extends ModelRecommendation { * Registry of available models with their hardware requirements */ const MODEL_REGISTRY: ModelRegistryEntry[] = [ - // GPT-OSS 21B - excellent quality open source model (only supported model for now) + // GPT-OSS 21B - excellent quality open source model for agentic wiki generation { family: 'gpt-oss', modelId: 'gpt-oss-20b-q8', @@ -43,6 +45,21 @@ const MODEL_REGISTRY: ModelRegistryEntry[] = [ contextLength: 32768, quality: 'excellent', }, + + // Qwen2.5-Coder-1.5B - small fast model for text completion (contextual retrieval) + // Specifically designed for code understanding, no function-calling tokens + { + family: 'completion', + modelId: 'qwen2.5-coder-1.5b-q8', + ggufFile: 'qwen2.5-coder-1.5b-instruct-q8_0.gguf', + downloadUrl: + 'https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF/resolve/main/qwen2.5-coder-1.5b-instruct-q8_0.gguf', + fileSizeBytes: 1_680_000_000, // ~1.68 GB + minVram: 2, + minRam: 4, + contextLength: 32768, + quality: 'good', + }, ]; export class ModelManager { diff --git a/src/llm/types.ts b/src/llm/types.ts index 338a151..aeb3789 100644 --- a/src/llm/types.ts +++ b/src/llm/types.ts @@ -195,8 +195,10 @@ export interface ModelRecommendation { /** * Model family type for local models + * - 'gpt-oss': Large agentic model (21B) with tool calling for wiki generation + * - 'completion': Small fast model (1.5B) for text completion tasks like contextual retrieval */ -export type ModelFamily = 'gpt-oss'; +export type ModelFamily = 'gpt-oss' | 'completion'; /** * Options for creating an LLM provider @@ -213,7 +215,7 @@ export interface CreateProviderOptions { // Local options modelPath?: string; localModel?: string; - modelFamily?: ModelFamily; // 'gpt-oss' (21B model) + modelFamily?: ModelFamily; // 'gpt-oss' (21B) or 'completion' (1.5B) gpuLayers?: number; contextSize?: number; threads?: number; diff --git a/src/mcp-wiki-server.ts b/src/mcp-wiki-server.ts index c67dac8..09b0ad3 100644 --- a/src/mcp-wiki-server.ts +++ b/src/mcp-wiki-server.ts @@ -1,5 +1,5 @@ /** - * MCP Server for querying ArchiWiki generated documentation + * MCP Server for querying SemanticWiki generated documentation * * This server exposes tools for: * - Searching wiki pages by keyword or semantic similarity @@ -49,7 +49,7 @@ export class WikiMCPServer { this.config = config; this.server = new Server( { - name: 'archiwiki', + name: 'semanticwiki', version: '1.0.0', }, { @@ -514,7 +514,7 @@ export class WikiMCPServer { const transport = new StdioServerTransport(); await this.server.connect(transport); - console.error('ArchiWiki MCP Server started'); + console.error('SemanticWiki MCP Server started'); } } @@ -536,7 +536,7 @@ async function main() { repoPath = args[++i]; } else if (args[i] === '--help' || args[i] === '-h') { console.log(` -ArchiWiki MCP Server +SemanticWiki MCP Server Usage: npx ts-node src/mcp-wiki-server.ts [options] diff --git a/src/package-format.ts b/src/package-format.ts index 518ead9..096828e 100644 --- a/src/package-format.ts +++ b/src/package-format.ts @@ -1,5 +1,5 @@ /** - * ArchiWiki Portable Package Format (.archiwiki) + * SemanticWiki Portable Package Format (.semantics) * * A portable package format that bundles: * - Wiki documentation (markdown files) @@ -53,7 +53,7 @@ export interface ExtractOptions { } /** - * Create an ArchiWiki package from a wiki directory + * Create a SemanticWiki package from a wiki directory */ export async function createPackage(options: PackageOptions): Promise { const { wikiPath, outputPath, name, description, sourceRepo, includeRag = true } = options; @@ -154,7 +154,7 @@ export async function createPackage(options: PackageOptions): Promise { // Format: [manifest_length (4 bytes)][manifest JSON][file entries...] // Each file entry: [path_length (2 bytes)][path][content_length (4 bytes)][content] - const packagePath = outputPath.endsWith('.archiwiki') ? outputPath : `${outputPath}.archiwiki`; + const packagePath = outputPath.endsWith('.semantics') ? outputPath : `${outputPath}.semantics`; // Build the package buffer const chunks: Buffer[] = []; @@ -204,7 +204,7 @@ export async function createPackage(options: PackageOptions): Promise { } /** - * Extract an ArchiWiki package to a directory + * Extract a SemanticWiki package to a directory */ export async function extractPackage(options: ExtractOptions): Promise { const { packagePath, outputPath, wikiOnly = false } = options; diff --git a/src/prompts/wiki-system.ts b/src/prompts/wiki-system.ts index a046bef..b235724 100644 --- a/src/prompts/wiki-system.ts +++ b/src/prompts/wiki-system.ts @@ -3,20 +3,25 @@ * * This is the most critical component - it defines the agent's identity, * capabilities, process, and output requirements. + * + * Inspired by DeepWiki's approach: hierarchical organization, deep understanding + * before generation, exhaustive code traceability, and rich component documentation. */ export const WIKI_SYSTEM_PROMPT = ` # ArchitecturalWiki Agent -You are ArchitecturalWiki, an expert software architect and technical documentation specialist. Your mission is to generate comprehensive, traceable architectural documentation for code repositories that bridges technical implementation with business domain understanding. +You are ArchitecturalWiki, an expert software architect and technical documentation specialist. Your mission is to generate comprehensive, traceable architectural documentation that rivals professional documentation platforms like DeepWiki. ## Core Identity -- You understand code at architectural levels: patterns, trade-offs, relationships +- You understand code at ARCHITECTURAL levels: patterns, trade-offs, relationships, system boundaries - You understand code at BUSINESS DOMAIN levels: what problems it solves, user workflows it supports +- You build a DEEP MENTAL MODEL of the entire system before writing any documentation - You write for developers who are new to a codebase - You prioritize clarity, accuracy, and practical utility -- You ALWAYS trace concepts back to source code +- You ALWAYS trace concepts back to source code with EXHAUSTIVE references - You ALWAYS explain the "why" alongside the "what" - the business purpose behind technical decisions +- You create HIERARCHICAL documentation that mirrors the system's logical architecture ## Business Domain Understanding (CRITICAL) Your documentation must bridge the gap between code and business value. For every component you document: @@ -27,16 +32,11 @@ Your documentation must bridge the gap between code and business value. For ever 4. **Domain Relationships**: How does this connect to other business domains in the system? When analyzing code: -- Look for domain-specific naming (e.g., "Invoice", "Cart", "Subscription" suggest business entities) -- Identify workflow patterns (authentication flow, checkout process, notification system) +- Look for domain-specific naming (e.g., "Invoice", "Cart", "Subscription", "Transaction", "Account") +- Identify workflow patterns (authentication flow, checkout process, batch processing, data synchronization) - Map technical components to business capabilities - Understand the data model from a business perspective, not just technical structure -Example domain context for an authentication service: -> "The AuthenticationService manages user identity verification, enabling secure access to the platform. -> It supports the business requirement for multi-factor authentication in high-security operations -> and integrates with the subscription system to enforce tier-based access controls." - ## Available Tools ### Filesystem Tools (via mcp__filesystem__) @@ -76,275 +76,517 @@ Example domain context for an authentication service: - \`mcp__semanticwiki__list_wiki_pages\`: List all created wiki pages - Use to see what pages already exist before creating new ones -## Generation Process +## Generation Process (DeepWiki-Quality) + +Follow this process for every wiki generation. The key difference from basic documentation is building a DEEP UNDERSTANDING before writing. + +### Phase 1: Deep Discovery (CRITICAL - DO NOT RUSH) + +This phase builds your mental model. Spend significant effort here. + +1. **Repository Structure Analysis** + - Use \`mcp__filesystem__directory_tree\` to understand the full project structure + - Identify the project type, framework, and technology stack + - Map out the directory organization pattern (by feature, by layer, hybrid, etc.) + +2. **Entry Point Analysis** + - Read package.json, README.md, main entry points (index.ts, main.py, etc.) + - Understand how the application starts and initializes + - Identify configuration and environment setup + +3. **System Architecture Discovery** + - Use \`mcp__semanticwiki__search_codebase\` to find: + - Controllers/handlers (entry points for user actions) + - Services (business logic layer) + - Data access layer (repositories, database queries) + - Models/entities (data structures) + - Middleware (cross-cutting concerns) + - Build a mental map of how data flows through the system -Follow this process for every wiki generation: +4. **Domain Model Analysis** + - Identify core business entities and their relationships + - Understand the data model and schema + - Map user workflows to code paths -### Phase 1: Discovery -1. Use \`mcp__filesystem__directory_tree\` to understand the project structure -2. Identify the project type (Node.js, Python, etc.), framework, and key directories -3. Read key files like package.json, README.md, or main entry points -4. Create a mental model of the architecture +5. **Technology Integration Points** + - Identify external services, databases, message queues + - Understand API boundaries and contracts + - Note configuration and deployment aspects -### Phase 2: Planning -1. Determine wiki structure based on codebase analysis -2. Identify major components/modules to document -3. Plan which diagrams are needed (architecture overview, data flow, etc.) -4. Decide on page hierarchy +### Phase 2: Hierarchical Planning -### Phase 3: Content Generation -For each wiki section: -1. Use \`mcp__semanticwiki__search_codebase\` to gather relevant code snippets -2. Use \`mcp__filesystem__read_file\` for detailed code examination -3. Use \`mcp__semanticwiki__analyze_code_structure\` for structure information -4. Generate documentation with PROPER SOURCE TRACEABILITY -5. Create supporting Mermaid diagrams using \`mcp__mermaid__generate_diagram\` -6. Write the wiki page using \`mcp__semanticwiki__write_wiki_page\` +Based on your deep understanding, plan a HIERARCHICAL wiki structure that mirrors the system architecture. + +**Structure Template (adapt based on project type):** + +\`\`\` +README.md (Overview + Navigation) +ā”‚ +ā”œā”€ā”€ System Architecture/ +ā”‚ ā”œā”€ā”€ overview.md (High-level architecture, diagrams) +ā”‚ ā”œā”€ā”€ core-technology-stack.md (Technologies, frameworks) +ā”‚ ā”œā”€ā”€ data-storage.md (Database, file organization) +ā”‚ ā””ā”€ā”€ integration-points.md (External services, APIs) +ā”‚ +ā”œā”€ā”€ [Domain Area 1]/ (e.g., "Online System", "User Management") +ā”‚ ā”œā”€ā”€ index.md (Domain overview) +ā”‚ ā”œā”€ā”€ [feature-1].md (Specific feature) +ā”‚ ā”œā”€ā”€ [feature-2].md +ā”‚ ā””ā”€ā”€ ... +ā”‚ +ā”œā”€ā”€ [Domain Area 2]/ (e.g., "Batch Processing", "Payment System") +ā”‚ ā”œā”€ā”€ index.md +ā”‚ ā”œā”€ā”€ [feature-1].md +ā”‚ ā””ā”€ā”€ ... +ā”‚ +ā”œā”€ā”€ Data Model/ +ā”‚ ā”œā”€ā”€ overview.md (Entity relationships, ERD) +ā”‚ ā””ā”€ā”€ [entity-name].md (For complex entities) +ā”‚ +ā””ā”€ā”€ Getting Started/ + ā”œā”€ā”€ setup.md (Installation, configuration) + ā””ā”€ā”€ development.md (Dev workflow, testing) +\`\`\` + +**For Mainframe/COBOL Projects, consider:** +\`\`\` +README.md +ā”œā”€ā”€ System Architecture/ +ā”‚ ā”œā”€ā”€ overview.md +ā”‚ ā”œā”€ā”€ technology-stack.md (COBOL, CICS, DB2, etc.) +ā”‚ ā””ā”€ā”€ data-organization.md (VSAM, datasets) +ā”‚ +ā”œā”€ā”€ Online System (CICS)/ +ā”‚ ā”œā”€ā”€ index.md +ā”‚ ā”œā”€ā”€ user-authentication.md +ā”‚ ā”œā”€ā”€ [transaction-type].md +ā”‚ ā””ā”€ā”€ ... +ā”‚ +ā”œā”€ā”€ Batch Processing/ +ā”‚ ā”œā”€ā”€ index.md +ā”‚ ā”œā”€ā”€ daily-processing.md +ā”‚ ā”œā”€ā”€ monthly-processing.md +ā”‚ ā””ā”€ā”€ job-scheduling.md +ā”‚ +ā”œā”€ā”€ JCL Jobs/ +ā”‚ ā”œā”€ā”€ index.md +ā”‚ ā””ā”€ā”€ [job-category].md +ā”‚ +ā””ā”€ā”€ Data Model/ + ā”œā”€ā”€ copybooks.md + ā””ā”€ā”€ file-definitions.md +\`\`\` + +### Phase 3: Content Generation (Exhaustive Traceability) + +For each wiki section, follow this rigorous process: + +1. **Gather ALL relevant code** using \`mcp__semanticwiki__search_codebase\` +2. **Read key files** with \`mcp__filesystem__read_file\` for detailed understanding +3. **Document with EXHAUSTIVE source references** - every claim must be traceable +4. **Create supporting diagrams** using \`mcp__mermaid__generate_diagram\` +5. **Write the page** using \`mcp__semanticwiki__write_wiki_page\` ### Phase 4: Cross-Referencing 1. Ensure all internal links between wiki pages resolve correctly 2. Add "Related" sections to connect pages -3. Generate the glossary/index page last +3. Generate the index/glossary page last ### Phase 5: Verification (MANDATORY) **You MUST complete this phase before finishing:** 1. Run \`mcp__semanticwiki__verify_wiki_completeness\` to check all internal links 2. If ANY broken links are found: - - Create each missing page immediately using \`mcp__semanticwiki__write_wiki_page\` - - Use \`mcp__semanticwiki__search_codebase\` to find relevant code for each missing topic + - Create each missing page immediately + - Use \`mcp__semanticwiki__search_codebase\` to find relevant code 3. Run verification again to confirm all links are valid 4. Repeat until verification shows 0 broken links -5. Only then is the wiki generation complete -## OUTPUT REQUIREMENTS (CRITICAL) +## OUTPUT REQUIREMENTS (DeepWiki Quality) + +### 1. Key Components Table (REQUIRED for all feature pages) + +Every page documenting a feature or module MUST include a Key Components table: + +\`\`\`markdown +## Key Components + +| Component | Purpose & Responsibility | How It Works (High-Level) | Source | +|-----------|-------------------------|---------------------------|--------| +| ComponentName | Brief description of what it does and why | 1-2 sentence explanation of mechanism | šŸ“„ [\`path/to/file.ts:23\`](../path/to/file.ts#L23) | +| AnotherComponent | What business need it addresses | How it accomplishes this | šŸ“„ [\`path/to/file.ts:45\`](../path/to/file.ts#L45) | +\`\`\` + +**Table Guidelines:** +- Include 5-15 components per page (be comprehensive) +- Sort by importance or logical flow order +- Use the šŸ“„ emoji before source links for visual scanning +- Include line numbers for precise navigation + +### 2. How It Works Section (REQUIRED) + +Every feature page MUST have a detailed "How It Works" section with numbered subsections: + +\`\`\`markdown +## How It Works + +### 2.1 [Workflow Name] Flow + +[Narrative description of the workflow] + +1. **Step Name** - Description of what happens + +\`\`\`language +// Relevant code snippet +\`\`\` +šŸ“„ [\`path/to/file.ts:23-45\`](../path/to/file.ts#L23-L45) + +2. **Next Step** - Description + +\`\`\`language +// Code +\`\`\` +šŸ“„ [\`path/to/file.ts:67-89\`](../path/to/file.ts#L67-L89) -### Source Traceability (NON-NEGOTIABLE) -EVERY architectural concept, pattern, or component MUST include source references. -This is the key differentiator of ArchitecturalWiki - all documentation traces back to code. +### 2.2 [Another Workflow] + +... +\`\`\` + +### 3. Source Traceability (NON-NEGOTIABLE) + +EVERY architectural concept, pattern, or component MUST include MULTIPLE source references. **Required Format:** \`\`\`markdown -## Authentication Flow +## Feature Name -The authentication system uses JWT tokens for stateless auth. +The authentication system uses JWT tokens for stateless auth across distributed services. -**Source:** [\`src/auth/jwt-provider.ts:23-67\`](../../../src/auth/jwt-provider.ts#L23-L67) +šŸ“„ **Primary Source:** [\`src/auth/jwt-provider.ts:23-67\`](../../../src/auth/jwt-provider.ts#L23-L67) \`\`\`typescript -// Relevant code snippet from the source +// Show the most important 5-30 lines export class JwtProvider { async generateToken(user: User): Promise { - // ... + return jwt.sign( + { userId: user.id, roles: user.roles }, + this.secret, + { expiresIn: '24h' } + ); } } \`\`\` + +**Related Sources:** +- šŸ“„ [\`src/auth/middleware.ts:12-34\`](../../../src/auth/middleware.ts#L12-L34) - Token validation middleware +- šŸ“„ [\`src/config/auth.ts:5-15\`](../../../src/config/auth.ts#L5-L15) - JWT configuration +- šŸ“„ [\`src/types/auth.ts:1-20\`](../../../src/types/auth.ts#L1-L20) - Type definitions \`\`\` -### Code Snippets -- Include relevant code snippets (5-30 lines typically) -- Always show the file path and line numbers in **Source:** tag -- Use syntax highlighting with correct language identifier -- Focus on the most important parts, not entire files +### 4. Mermaid Diagrams (COMPREHENSIVE) -### Mermaid Diagrams -- Use Mermaid format exclusively (rendered natively in GitHub/GitLab) -- Always wrap in \`\`\`mermaid code blocks -- Include descriptive labels on all nodes and edges -- Keep diagrams focused - split large diagrams into multiple smaller ones -- Use appropriate diagram types: - - \`flowchart\` for architecture and data flow - - \`sequenceDiagram\` for interactions between components - - \`classDiagram\` for object relationships - - \`erDiagram\` for data models - -### Page Structure -Every wiki page MUST include: -1. **Frontmatter with title** - Title in YAML frontmatter (do NOT repeat as H1 in content) -2. **Brief description** - 1-2 sentences explaining what this page covers -3. **Business Context section** - What business problem this solves, who uses it, why it exists -4. **Overview section** - High-level summary with key files listed -5. **Detailed content** - With source references for every concept -6. **Domain Relationships** - How this connects to other business domains/workflows -7. **Related pages** - Links to connected documentation -8. **Source files list** - At bottom, list all files referenced - -### Business Context Template -For each major component, include a "Business Context" section like: -\`\`\`markdown -## Business Context +Diagrams are CRITICAL for understanding system relationships. Use Mermaid format exclusively. + +**Required Diagrams by Page Type:** -**Business Problem**: [What user/business need does this address?] +**Architecture Overview Page:** +\`\`\`mermaid +flowchart TB + subgraph "Presentation Layer" + UI[User Interface] + API[API Gateway] + end + subgraph "Business Logic" + Auth[Authentication] + Services[Core Services] + end + subgraph "Data Layer" + DB[(Database)] + Cache[(Cache)] + end + UI --> API --> Auth --> Services --> DB + Services --> Cache +\`\`\` +*Caption: High-level system architecture showing layer separation* -**User Impact**: [How do end users interact with or benefit from this?] +**Feature/Module Pages - Include ALL of these:** -**Workflow Role**: [Where does this fit in the overall user journey/workflow?] +1. **Component Interaction Diagram** (flowchart): +\`\`\`mermaid +flowchart LR + A[Entry Point] --> B[Validator] + B --> C{Decision} + C -->|Valid| D[Processor] + C -->|Invalid| E[Error Handler] + D --> F[Repository] + F --> G[(Database)] \`\`\` -## Wiki Structure +2. **Sequence Diagram** (for workflows): +\`\`\`mermaid +sequenceDiagram + participant U as User + participant C as Controller + participant S as Service + participant D as Database + U->>C: Request + C->>S: Process + S->>D: Query + D-->>S: Result + S-->>C: Response + C-->>U: Display +\`\`\` -Generate pages in this order: +3. **Data Flow Diagram** (for data-heavy features): +\`\`\`mermaid +flowchart LR + Input[/"Input Data"/] --> Transform["Transform"] + Transform --> Validate{"Validate"} + Validate -->|Pass| Store[("Store")] + Validate -->|Fail| Reject["Reject"] + Store --> Output[/"Output"/] +\`\`\` -1. **README.md** - Entry point with: - - Project overview (from actual README if exists) - - Navigation tree to all wiki sections - - Quick links to most important pages +**Data Model Pages:** +\`\`\`mermaid +erDiagram + USER ||--o{ ORDER : places + ORDER ||--|{ LINE_ITEM : contains + PRODUCT ||--o{ LINE_ITEM : "ordered in" + USER { + int id PK + string email + string name + } + ORDER { + int id PK + int user_id FK + date created_at + } +\`\`\` -2. **architecture/overview.md** - High-level system design with: - - Architecture diagram (Mermaid) - - Key design decisions - - Technology stack - - Directory structure explanation +**Batch/Job Processing Pages:** +\`\`\`mermaid +flowchart TB + subgraph "Job Scheduler" + Trigger[Scheduled Trigger] + end + subgraph "Job Execution" + Step1[Step 1: Extract] + Step2[Step 2: Transform] + Step3[Step 3: Load] + end + subgraph "Resources" + Input[(Input File)] + Output[(Output DB)] + end + Trigger --> Step1 + Input --> Step1 --> Step2 --> Step3 --> Output +\`\`\` -3. **architecture/data-flow.md** - How data moves through system: - - Request/response lifecycle - - Data transformation points - - Sequence diagrams for key flows +**State Machine Diagrams** (for workflows with states): +\`\`\`mermaid +stateDiagram-v2 + [*] --> Pending + Pending --> Processing: start + Processing --> Completed: success + Processing --> Failed: error + Failed --> Pending: retry + Completed --> [*] +\`\`\` -4. **Component pages** - One per major module: - - Located in components/{module-name}/index.md - - Each with its own architecture and source refs +**Diagram Requirements:** +- EVERY feature page needs at least 2 diagrams (component interaction + workflow) +- Architecture pages need at least 3 diagrams (overview, data flow, integration) +- Always wrap in \`\`\`mermaid code blocks +- Include descriptive labels on all nodes and edges +- Use subgraphs to show logical groupings +- Add a caption below each diagram explaining what it shows +- Keep diagrams focused - split large diagrams into multiple smaller ones +- Use consistent naming (match code names where possible) -5. **guides/getting-started.md** - Quick start for new devs: - - How to run locally - - Key files to understand first - - Common modification patterns +**Cross-Reference Diagrams:** +When components interact across pages, include a diagram showing the integration: +\`\`\`mermaid +flowchart LR + subgraph "This Module" + A[Component A] + end + subgraph "Related Module" + B[Component B] + end + A -->|"calls"| B + click B "./related-page.md" "See Related Module docs" +\`\`\` -6. **glossary.md** - Concept index: - - Alphabetical list of key terms - - Each links to the page where it's explained +### 5. Page Structure Template -## Example Page Output +Every wiki page MUST follow this structure: \`\`\`markdown --- -title: Authentication System -generated: 2025-01-15T10:30:00Z -description: Secure user identity management using JWT tokens +title: Feature Name +generated: [ISO timestamp] +description: 1-2 sentence description sources: - - src/auth/index.ts - - src/auth/jwt-provider.ts - - src/auth/oauth/ + - path/to/main/file.ts + - path/to/related/file.ts related: - - api/middleware.md - - components/session.md + - ./related-page.md + - ../other-section/page.md --- -The authentication system provides secure user identity management using JWT tokens and supports multiple OAuth providers. +[1-2 paragraph introduction explaining what this page covers and why it matters] ## Business Context -**Business Problem**: Users need secure access to the platform with varying permission levels. The business requires audit trails for compliance and support for enterprise SSO. - -**User Impact**: End users experience seamless login via email/password or social accounts. Enterprise customers can use their corporate identity providers. +**Business Problem**: What user/business need does this address? -**Workflow Role**: Authentication is the gateway to all protected features. It validates identity before checkout, profile management, and admin operations. - -## Overview +**User Impact**: How do end users interact with or benefit from this? -This module handles: -- User login/logout flows -- JWT token generation and validation -- OAuth2 integration (Google, GitHub) -- Session management +**Workflow Role**: Where does this fit in the overall user journey? -**Key Files:** -- \`src/auth/index.ts\` - Main exports -- \`src/auth/jwt-provider.ts\` - Token management -- \`src/auth/oauth/\` - OAuth provider implementations +## Architecture Overview -## Architecture +[Brief architecture description] \`\`\`mermaid flowchart LR - Client --> AuthController - AuthController --> JwtProvider - AuthController --> OAuthHandler - JwtProvider --> TokenStore - OAuthHandler --> GoogleProvider - OAuthHandler --> GitHubProvider + ... \`\`\` -## JWT Token Flow +## Key Components -The JWT provider handles token lifecycle management, enabling stateless authentication across the distributed system. +| Component | Purpose & Responsibility | How It Works | Source | +|-----------|-------------------------|--------------|--------| +| ... | ... | ... | ... | -**Source:** [\`src/auth/jwt-provider.ts:23-45\`](../../../src/auth/jwt-provider.ts#L23-L45) +## How It Works -\`\`\`typescript -export class JwtProvider { - private readonly secret: string; +### 2.1 [Primary Workflow] - async generateToken(user: User): Promise { - return jwt.sign( - { userId: user.id, roles: user.roles }, - this.secret, - { expiresIn: '24h' } - ); - } -} -\`\`\` +[Detailed walkthrough with code snippets and source links] + +### 2.2 [Secondary Workflow] + +... + +## Data Model + +[If applicable - describe data structures used] + +## Integration Points + +[If applicable - describe how this connects to other systems/modules] -The token includes the user ID and roles, enabling stateless authorization checks. +## Related Documentation -## Domain Relationships +- [Link to related page 1](./related.md) +- [Link to related page 2](../other/page.md) -- **Subscription System**: Token claims include subscription tier for feature gating -- **Audit Logging**: All authentication events are logged for compliance -- **Session Management**: Coordinates with session service for device tracking +--- +**Source Files Referenced:** +- path/to/file1.ts:lines +- path/to/file2.ts:lines +- path/to/file3.ts:lines +\`\`\` -## Related Pages -- [Session Management](./session.md) -- [OAuth Providers](./oauth/index.md) -- [API Authentication Middleware](../api/middleware.md) +### 6. README.md Structure (Navigation Hub) +The README must serve as a navigation hub with: + +\`\`\`markdown --- -**Sources:** -- src/auth/index.ts -- src/auth/jwt-provider.ts:23-45 -- src/auth/types.ts +title: [Project Name] Wiki +--- + +[Project description from README or inferred] + +## Overview + +[High-level description of what the project does, its purpose, and key capabilities] + +## Navigation + +### System Architecture +- [Architecture Overview](./architecture/overview.md) +- [Technology Stack](./architecture/technology-stack.md) +- [Data Storage](./architecture/data-storage.md) + +### [Domain Area 1] (e.g., Online System) +- [Overview](./domain1/index.md) +- [Feature A](./domain1/feature-a.md) +- [Feature B](./domain1/feature-b.md) + +### [Domain Area 2] (e.g., Batch Processing) +- [Overview](./domain2/index.md) +- [Feature C](./domain2/feature-c.md) + +### Data Model +- [Entity Relationships](./data-model/overview.md) + +### Getting Started +- [Setup Guide](./getting-started/setup.md) +- [Development Guide](./getting-started/development.md) + +## Quick Reference + +| Component | Description | Documentation | +|-----------|-------------|---------------| +| [Name] | Brief description | [Link](./path.md) | +| ... | ... | ... | \`\`\` ## Quality Checklist Before marking generation complete, verify: -- [ ] Every architectural concept has source file references + +**Content Quality:** +- [ ] Every page has a Key Components table with 5+ entries +- [ ] Every page has a "How It Works" section with numbered subsections +- [ ] Every architectural concept has 2+ source file references - [ ] Every major component has a Business Context section -- [ ] Domain relationships are documented for key modules +- [ ] Code snippets have language identifiers and are 5-30 lines + +**Diagram Quality:** +- [ ] Architecture overview has 3+ diagrams (system overview, data flow, integration) +- [ ] Every feature page has 2+ diagrams (component interaction + workflow) - [ ] All Mermaid diagrams use valid syntax +- [ ] Every diagram has a descriptive caption below it +- [ ] Complex systems have sequence diagrams showing interactions +- [ ] Data-heavy features have ER diagrams or data flow diagrams + +**Navigation & Links:** +- [ ] README.md has hierarchical navigation linking ALL pages - [ ] Internal links use correct relative paths -- [ ] Code snippets have language identifiers -- [ ] README.md links to all generated pages - [ ] No orphan pages (all reachable from README) -- [ ] No duplicate H1 titles (title should only be in frontmatter) +- [ ] Cross-references between related pages exist +- [ ] No duplicate H1 titles (title only in frontmatter) - [ ] **CRITICAL:** \`mcp__semanticwiki__verify_wiki_completeness\` returns 0 broken links ## Important Notes -1. **Be thorough** - Read enough code to truly understand the architecture -2. **Be accurate** - Only document what you've verified in the code -3. **Be practical** - Focus on what developers need to know -4. **Be consistent** - Use the same format and style throughout -5. **Source everything** - If you can't find a source reference, don't include the claim -6. **Business focus** - Always explain the business purpose, not just technical implementation -7. **No hallucination** - Base ALL documentation on actual code analysis, embeddings, and domain hints from the indexer +1. **Deep understanding first** - Spend 40% of effort understanding, 60% writing +2. **Be exhaustive** - Document ALL major components, not just the obvious ones +3. **Be accurate** - Only document what you've verified in the code +4. **Be traceable** - Every claim needs a source reference +5. **Be hierarchical** - Mirror the system's logical architecture in page structure +6. **No hallucination** - Base ALL documentation on actual code analysis +7. **Quality over quantity** - A smaller wiki with exhaustive traceability beats a large wiki with sparse references ## CRITICAL: Complete All Pages **YOU MUST GENERATE ALL PAGES YOU REFERENCE IN THE README.** -If your README.md contains a link to a page like \`components/auth/index.md\`, you MUST create that file before finishing. +If your README.md contains a link to a page, you MUST create that file before finishing. Follow this workflow strictly: 1. First, analyze the codebase and plan ALL pages you will create 2. Create the README.md with links to all planned pages 3. **THEN, generate EVERY page linked in the README** - do not stop until all pages exist -4. If you run low on context or time, prioritize creating stub pages with basic structure over skipping pages entirely - -After writing README.md, immediately check: "Did I link to pages that don't exist yet?" If yes, create them NOW. +4. If you run low on context, prioritize creating pages with basic structure over skipping -A wiki with broken links is worse than a smaller wiki with complete pages. Either: -- Create all the pages you link to, OR -- Only link to pages you will actually create +A wiki with broken links is worse than a smaller wiki with complete pages. ## FINAL VERIFICATION LOOP (NON-NEGOTIABLE) diff --git a/src/rag/contextual-retrieval.ts b/src/rag/contextual-retrieval.ts new file mode 100644 index 0000000..6491819 --- /dev/null +++ b/src/rag/contextual-retrieval.ts @@ -0,0 +1,709 @@ +/** + * Contextual Retrieval for SemanticWiki + * + * Implements Anthropic's Contextual Retrieval technique to improve RAG quality. + * For each chunk, generates a brief context explanation using the full file content. + * + * Supports three modes: + * - Claude API (with prompt caching for cost efficiency) + * - Ollama (external local server) + * - Bundled local (node-llama-cpp, no external dependencies) + * + * Reference: https://www.anthropic.com/news/contextual-retrieval + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import Anthropic from '@anthropic-ai/sdk'; +import type { ASTChunk } from '../ast-chunker.js'; +import type { LLMProvider } from '../llm/types.js'; + +/** + * Configuration for contextual retrieval + */ +export interface ContextualRetrievalConfig { + /** Enable contextual retrieval (default: false) */ + enabled: boolean; + /** Use local LLM instead of Claude API */ + useLocal?: boolean; + /** Use Ollama server (requires useLocal: true) */ + useOllama?: boolean; + /** Anthropic API key (for Claude API mode) */ + apiKey?: string; + /** Claude model to use (default: claude-3-haiku for speed/cost) */ + model?: string; + /** Ollama host URL (for Ollama mode) */ + ollamaHost?: string; + /** Local model name (for Ollama or bundled local) */ + localModel?: string; + /** Model family for bundled local (default: gpt-oss) */ + modelFamily?: string; + /** Maximum concurrent requests (default: 5 for API, 1 for local) */ + concurrency?: number; + /** Cache generated contexts to avoid regeneration */ + cacheDir?: string; + /** Progress callback */ + onProgress?: (current: number, total: number) => void; +} + +/** + * Result of contextual enrichment + */ +export interface ContextualChunk extends ASTChunk { + /** Generated context explaining the chunk within the file */ + contextualPrefix: string; + /** Combined content for embedding (context + original) */ + enrichedContent: string; +} + +/** + * Context generation prompt template + * Based on Anthropic's recommended approach + */ +const CONTEXT_PROMPT = ` +{WHOLE_DOCUMENT} + + +Here is the chunk we want to situate within the whole document: + +{CHUNK_CONTENT} + + +Please give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk. The context should: +1. Identify what file/module this is from and its purpose +2. Explain what this specific code does in the context of the whole file +3. Note any important relationships to other parts of the codebase + +Answer only with the context, nothing else. Keep it under 100 words.`; + +/** + * Contextual Retrieval Generator + * + * Enriches code chunks with contextual information for better retrieval. + */ +export class ContextualRetrieval { + private config: Required; + private anthropicClient?: Anthropic; + private localProvider?: LLMProvider; + private contextCache: Map = new Map(); + private fileContentCache: Map = new Map(); + + constructor(config: ContextualRetrievalConfig) { + this.config = { + enabled: config.enabled, + useLocal: config.useLocal ?? false, + useOllama: config.useOllama ?? false, + apiKey: config.apiKey ?? process.env.ANTHROPIC_API_KEY ?? '', + model: config.model ?? 'claude-3-haiku-20240307', + ollamaHost: config.ollamaHost ?? 'http://localhost:11434', + localModel: config.localModel ?? 'qwen2.5-coder:7b', + modelFamily: config.modelFamily ?? 'gpt-oss', + concurrency: config.concurrency ?? (config.useLocal ? 1 : 5), + cacheDir: config.cacheDir ?? '', + onProgress: config.onProgress ?? (() => {}), + }; + + // Load cache if available + if (this.config.cacheDir) { + this.loadCache(); + } + } + + /** + * Initialize the contextual retrieval system + */ + async initialize(): Promise { + if (!this.config.enabled) return; + + if (!this.config.useLocal) { + // Initialize Anthropic client + if (!this.config.apiKey) { + throw new Error('Anthropic API key required for contextual retrieval. Set ANTHROPIC_API_KEY or use --contextual-local'); + } + this.anthropicClient = new Anthropic({ apiKey: this.config.apiKey }); + console.log(' Using Claude API for contextual retrieval'); + } else if (this.config.useOllama) { + // Verify Ollama is available + try { + const response = await fetch(`${this.config.ollamaHost}/api/tags`); + if (!response.ok) { + throw new Error(`Ollama not available at ${this.config.ollamaHost}`); + } + console.log(` Using Ollama (${this.config.localModel}) for contextual retrieval`); + } catch (error) { + throw new Error(`Cannot connect to Ollama at ${this.config.ollamaHost}: ${(error as Error).message}`); + } + } else { + // Use bundled local inference (node-llama-cpp) + // Use the 'completion' model family (Qwen2.5-Coder-1.5B) instead of gpt-oss + // because gpt-oss has function-calling tokens that cause empty responses + // for simple text completion tasks + console.log(' Initializing bundled local LLM for contextual retrieval...'); + try { + const { createLLMProvider } = await import('../llm/index.js'); + // Use 'completion' family - smaller model optimized for text completion + this.localProvider = await createLLMProvider({ + fullLocal: true, + modelFamily: 'completion', + }); + console.log(` Using bundled local LLM (Qwen2.5-Coder) for contextual retrieval`); + } catch (error) { + throw new Error(`Failed to initialize local LLM: ${(error as Error).message}`); + } + } + } + + /** + * Enrich chunks with contextual information + * + * @param chunks - Code chunks to enrich + * @param repoPath - Path to the repository + * @returns Enriched chunks with contextual prefixes + */ + async enrichChunks( + chunks: ASTChunk[], + repoPath: string + ): Promise { + if (!this.config.enabled) { + // Return chunks as-is with empty context + return chunks.map(chunk => ({ + ...chunk, + contextualPrefix: '', + enrichedContent: chunk.content, + })); + } + + console.log(` Generating contextual enrichment for ${chunks.length} chunks...`); + + // Group chunks by file for efficient processing + const chunksByFile = new Map(); + for (const chunk of chunks) { + const existing = chunksByFile.get(chunk.filePath) || []; + existing.push(chunk); + chunksByFile.set(chunk.filePath, existing); + } + + const enrichedChunks: ContextualChunk[] = []; + let processedCount = 0; + let successCount = 0; + let emptyCount = 0; + let fallbackCount = 0; + let sequenceResetCount = 0; + + // For bundled local mode, we need to reinitialize periodically to avoid "No sequences left" error + // The node-llama-cpp context has limited sequences (20), and they don't get released fast enough + const SEQUENCE_RESET_THRESHOLD = 15; // Reset before hitting the 20 sequence limit + let localChunksSinceReset = 0; + + // Process files in batches (or sequentially for local) + const fileEntries = Array.from(chunksByFile.entries()); + const concurrency = this.config.useLocal && !this.config.useOllama ? 1 : this.config.concurrency; + + // Helper to reinitialize local provider when sequences are exhausted + const reinitializeLocalProvider = async () => { + if (this.localProvider && this.config.useLocal && !this.config.useOllama) { + try { + // Shutdown existing provider + if ('shutdown' in this.localProvider) { + await (this.localProvider as any).shutdown(); + } + // Reinitialize with 'completion' family (Qwen2.5-Coder-1.5B) + const { createLLMProvider } = await import('../llm/index.js'); + this.localProvider = await createLLMProvider({ + fullLocal: true, + modelFamily: 'completion', + }); + localChunksSinceReset = 0; + sequenceResetCount++; + } catch (error) { + console.warn(` Warning: Failed to reinitialize local provider: ${(error as Error).message}`); + } + } + }; + + for (let i = 0; i < fileEntries.length; i += concurrency) { + const batch = fileEntries.slice(i, i + concurrency); + + const batchPromises = batch.map(async ([filePath, fileChunks]) => { + // Read file content (with caching) + let fileContent = this.fileContentCache.get(filePath); + if (!fileContent) { + try { + const fullPath = path.join(repoPath, filePath); + fileContent = fs.readFileSync(fullPath, 'utf-8'); + this.fileContentCache.set(filePath, fileContent); + } catch { + // If file can't be read, use chunk content as context + fileContent = fileChunks.map(c => c.content).join('\n\n'); + } + } + + // Generate context for each chunk in this file + const results: ContextualChunk[] = []; + for (const chunk of fileChunks) { + const cacheKey = `${chunk.id}:${chunk.content.length}`; + + // Check cache first + let contextualPrefix = this.contextCache.get(cacheKey); + let usedFallback = false; + + if (!contextualPrefix) { + // For bundled local mode, check if we need to reset sequences + if (this.config.useLocal && !this.config.useOllama) { + if (localChunksSinceReset >= SEQUENCE_RESET_THRESHOLD) { + console.log(` Resetting local LLM context (sequence limit reached)...`); + await reinitializeLocalProvider(); + } + localChunksSinceReset++; + } + + try { + contextualPrefix = await this.generateContext(fileContent, chunk); + + // If LLM returned empty, use fallback + if (!contextualPrefix || contextualPrefix.trim().length === 0) { + emptyCount++; + contextualPrefix = this.generateFallbackContext(chunk); + usedFallback = true; + } else { + successCount++; + } + + this.contextCache.set(cacheKey, contextualPrefix); + } catch (error) { + const errorMsg = (error as Error).message; + // If we hit sequence limit, try to recover + if (errorMsg.includes('No sequences left')) { + console.log(` Recovering from sequence exhaustion...`); + await reinitializeLocalProvider(); + // Retry once after reset + try { + contextualPrefix = await this.generateContext(fileContent, chunk); + if (!contextualPrefix || contextualPrefix.trim().length === 0) { + emptyCount++; + contextualPrefix = this.generateFallbackContext(chunk); + usedFallback = true; + } else { + successCount++; + } + this.contextCache.set(cacheKey, contextualPrefix); + } catch { + contextualPrefix = this.generateFallbackContext(chunk); + usedFallback = true; + fallbackCount++; + } + } else { + console.warn(` Warning: Failed to generate context for ${chunk.id}: ${errorMsg}`); + contextualPrefix = this.generateFallbackContext(chunk); + usedFallback = true; + fallbackCount++; + } + } + } + + results.push({ + ...chunk, + contextualPrefix, + enrichedContent: `${contextualPrefix}\n\n${chunk.content}`, + }); + + processedCount++; + this.config.onProgress(processedCount, chunks.length); + } + + return results; + }); + + const batchResults = await Promise.all(batchPromises); + for (const results of batchResults) { + enrichedChunks.push(...results); + } + + // Log progress + const progress = Math.round((processedCount / chunks.length) * 100); + console.log(` Contextual enrichment: ${processedCount}/${chunks.length} (${progress}%)`); + } + + // Log summary stats + console.log(` Contextual enrichment complete:`); + console.log(` āœ“ ${successCount} chunks enriched by LLM`); + if (emptyCount > 0) { + console.log(` āš  ${emptyCount} empty responses (fallback used)`); + } + if (fallbackCount > 0) { + console.log(` āœ— ${fallbackCount} errors (fallback used)`); + } + if (sequenceResetCount > 0) { + console.log(` šŸ”„ ${sequenceResetCount} context resets (sequence management)`); + } + + // Save cache + if (this.config.cacheDir) { + this.saveCache(); + } + + return enrichedChunks; + } + + /** + * Generate context for a single chunk using the LLM + */ + private async generateContext( + fileContent: string, + chunk: ASTChunk + ): Promise { + const prompt = CONTEXT_PROMPT + .replace('{WHOLE_DOCUMENT}', this.truncateContent(fileContent, 8000)) + .replace('{CHUNK_CONTENT}', this.truncateContent(chunk.content, 2000)); + + if (!this.config.useLocal) { + return this.generateContextClaude(prompt, fileContent); + } else if (this.config.useOllama) { + return this.generateContextOllama(prompt); + } else { + return this.generateContextBundledLocal(prompt); + } + } + + /** + * Generate context using Claude API with prompt caching + */ + private async generateContextClaude( + prompt: string, + _fileContent: string + ): Promise { + if (!this.anthropicClient) { + throw new Error('Anthropic client not initialized'); + } + + try { + const response = await this.anthropicClient.messages.create({ + model: this.config.model, + max_tokens: 150, + messages: [ + { + role: 'user', + content: prompt, + }, + ], + }); + + const textBlock = response.content.find(block => block.type === 'text'); + return textBlock && 'text' in textBlock ? textBlock.text.trim() : ''; + } catch (error) { + throw new Error(`Claude API error: ${(error as Error).message}`); + } + } + + /** + * Generate context using Ollama + */ + private async generateContextOllama(prompt: string): Promise { + try { + const response = await fetch(`${this.config.ollamaHost}/api/generate`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: this.config.localModel, + prompt: prompt, + stream: false, + options: { + num_predict: 150, + temperature: 0.3, + }, + }), + }); + + if (!response.ok) { + throw new Error(`Ollama error: ${response.statusText}`); + } + + const data = await response.json() as { response: string }; + return data.response?.trim() || ''; + } catch (error) { + throw new Error(`Ollama error: ${(error as Error).message}`); + } + } + + /** + * Generate context using bundled local LLM (node-llama-cpp) + */ + private async generateContextBundledLocal(prompt: string): Promise { + if (!this.localProvider) { + throw new Error('Local LLM provider not initialized'); + } + + // System prompt to guide the local model + const systemPrompt = `You are a code documentation assistant. Your task is to write a brief context description (under 100 words) that explains what a code chunk does within its file. Be concise and factual. Output ONLY the context description, nothing else.`; + + try { + const response = await this.localProvider.chat( + [{ role: 'user', content: prompt }], + [], // No tools needed for simple completion + { + maxTokens: 150, + temperature: 0.3, + systemPrompt, + } + ); + + return response.content.trim(); + } catch (error) { + throw new Error(`Local LLM error: ${(error as Error).message}`); + } + } + + /** + * Generate fallback context when LLM is unavailable + * Uses AST metadata to create a basic context + */ + private generateFallbackContext(chunk: ASTChunk): string { + const parts: string[] = []; + + // File info + const fileName = path.basename(chunk.filePath); + parts.push(`This code is from ${fileName}.`); + + // Chunk type and name + if (chunk.chunkType && chunk.name) { + const typeMap: Record = { + 'function': 'function', + 'class': 'class', + 'method': 'method', + 'interface': 'interface', + 'service': 'service', + 'controller': 'controller', + 'handler': 'handler', + 'model': 'data model', + 'repository': 'data access', + }; + const type = typeMap[chunk.chunkType] || chunk.chunkType; + parts.push(`It defines ${chunk.name}, a ${type}.`); + } + + // Parent context + if (chunk.parentName) { + parts.push(`It belongs to ${chunk.parentName}.`); + } + + // Domain hints + if (chunk.domainHints && chunk.domainHints.length > 0) { + const domains = chunk.domainHints + .filter(h => h.confidence > 0.5) + .map(h => h.category.replace('-', ' ')) + .slice(0, 2); + if (domains.length > 0) { + parts.push(`Related to ${domains.join(' and ')}.`); + } + } + + return parts.join(' '); + } + + /** + * Truncate content to fit within token limits + */ + private truncateContent(content: string, maxChars: number): string { + if (content.length <= maxChars) return content; + + // Truncate at a natural boundary (newline) + const truncated = content.slice(0, maxChars); + const lastNewline = truncated.lastIndexOf('\n'); + if (lastNewline > maxChars * 0.8) { + return truncated.slice(0, lastNewline) + '\n... [truncated]'; + } + return truncated + '\n... [truncated]'; + } + + /** + * Load context cache from disk + */ + private loadCache(): void { + const cachePath = path.join(this.config.cacheDir, 'contextual-cache.json'); + if (fs.existsSync(cachePath)) { + try { + const data = JSON.parse(fs.readFileSync(cachePath, 'utf-8')); + this.contextCache = new Map(Object.entries(data)); + console.log(` Loaded ${this.contextCache.size} cached contexts`); + } catch { + // Ignore cache errors + } + } + } + + /** + * Save context cache to disk + */ + private saveCache(): void { + if (!this.config.cacheDir) return; + + const cachePath = path.join(this.config.cacheDir, 'contextual-cache.json'); + try { + fs.mkdirSync(this.config.cacheDir, { recursive: true }); + const data = Object.fromEntries(this.contextCache); + fs.writeFileSync(cachePath, JSON.stringify(data, null, 2)); + } catch { + // Ignore cache errors + } + } + + /** + * Preview contextual enrichment for a sample of chunks + * Useful for debugging and validating the enrichment quality + */ + async previewEnrichment( + chunks: ASTChunk[], + repoPath: string, + sampleSize: number = 10 + ): Promise<{ + mode: string; + totalChunks: number; + sampleSize: number; + samples: Array<{ + id: string; + filePath: string; + chunkType?: string; + name?: string; + contentPreview: string; + contextualPrefix: string; + status: 'success' | 'empty' | 'fallback' | 'error'; + }>; + }> { + // Determine mode + let mode = 'claude-api'; + if (this.config.useLocal) { + mode = this.config.useOllama ? 'ollama' : 'bundled-local'; + } + + // Sample chunks (spread across different files) + const sampleChunks: ASTChunk[] = []; + const filesSeen = new Set(); + + for (const chunk of chunks) { + if (sampleChunks.length >= sampleSize) break; + if (!filesSeen.has(chunk.filePath) || sampleChunks.length < sampleSize / 2) { + sampleChunks.push(chunk); + filesSeen.add(chunk.filePath); + } + } + + const samples: Array<{ + id: string; + filePath: string; + chunkType?: string; + name?: string; + contentPreview: string; + contextualPrefix: string; + status: 'success' | 'empty' | 'fallback' | 'error'; + }> = []; + + for (const chunk of sampleChunks) { + let contextualPrefix = ''; + let status: 'success' | 'empty' | 'fallback' | 'error' = 'success'; + + // Read file content + let fileContent = ''; + try { + const fullPath = path.join(repoPath, chunk.filePath); + fileContent = fs.readFileSync(fullPath, 'utf-8'); + } catch { + fileContent = chunk.content; + } + + try { + contextualPrefix = await this.generateContext(fileContent, chunk); + + if (!contextualPrefix || contextualPrefix.trim().length === 0) { + status = 'empty'; + contextualPrefix = this.generateFallbackContext(chunk); + } + } catch (error) { + status = 'error'; + contextualPrefix = this.generateFallbackContext(chunk); + } + + samples.push({ + id: chunk.id, + filePath: chunk.filePath, + chunkType: chunk.chunkType, + name: chunk.name, + contentPreview: chunk.content.slice(0, 100) + (chunk.content.length > 100 ? '...' : ''), + contextualPrefix, + status, + }); + } + + return { + mode, + totalChunks: chunks.length, + sampleSize: samples.length, + samples, + }; + } + + /** + * Get statistics about contextual enrichment + */ + getStats(): { + cacheHits: number; + cacheSize: number; + filesProcessed: number; + } { + return { + cacheHits: this.contextCache.size, + cacheSize: this.contextCache.size, + filesProcessed: this.fileContentCache.size, + }; + } + + /** + * Cleanup resources + */ + async cleanup(): Promise { + if (this.localProvider && 'cleanup' in this.localProvider) { + await (this.localProvider as any).cleanup(); + } + } +} + +/** + * Estimate cost of contextual retrieval for a codebase + * + * @param chunkCount - Number of chunks to process + * @param avgFileSize - Average file size in characters + * @returns Estimated cost in USD + */ +export function estimateContextualRetrievalCost( + chunkCount: number, + avgFileSize: number = 5000 +): { + inputTokens: number; + outputTokens: number; + estimatedCost: number; + withCaching: number; +} { + // Estimate tokens (roughly 4 chars per token) + const tokensPerChunk = Math.ceil(avgFileSize / 4) + 500; // file + chunk + prompt + const outputTokensPerChunk = 100; + + const inputTokens = chunkCount * tokensPerChunk; + const outputTokens = chunkCount * outputTokensPerChunk; + + // Claude 3 Haiku pricing (as of 2024) + // Input: $0.25 per million tokens + // Output: $1.25 per million tokens + const inputCost = (inputTokens / 1_000_000) * 0.25; + const outputCost = (outputTokens / 1_000_000) * 1.25; + const estimatedCost = inputCost + outputCost; + + // With prompt caching, input costs are reduced by ~90% + // (same file content cached across chunks) + const withCaching = (inputCost * 0.1) + outputCost; + + return { + inputTokens, + outputTokens, + estimatedCost, + withCaching, + }; +} diff --git a/src/rag/index.ts b/src/rag/index.ts index 8794253..2ef1586 100644 --- a/src/rag/index.ts +++ b/src/rag/index.ts @@ -22,6 +22,11 @@ import { DomainCategory, generateDomainContext } from '../ast-chunker.js'; +import { + ContextualRetrieval, + type ContextualRetrievalConfig, + type ContextualChunk +} from './contextual-retrieval.js'; // Configure transformers.js to use local cache env.cacheDir = './.semanticwiki-models'; @@ -58,6 +63,20 @@ export interface RAGConfig { useReranking?: boolean; /** RRF constant k for rank fusion (default: 60) */ rrfK?: number; + /** Enable contextual retrieval for improved chunk understanding (default: false) */ + useContextualRetrieval?: boolean; + /** Use local LLM for contextual retrieval instead of Claude API */ + contextualLocal?: boolean; + /** Use Ollama server for contextual retrieval (requires contextualLocal: true) */ + contextualUseOllama?: boolean; + /** Anthropic API key for contextual retrieval (falls back to ANTHROPIC_API_KEY env) */ + contextualApiKey?: string; + /** Model for contextual retrieval (default: claude-3-haiku-20240307) */ + contextualModel?: string; + /** Ollama host for local contextual retrieval */ + contextualOllamaHost?: string; + /** Local model name for contextual retrieval */ + contextualLocalModel?: string; } export interface BatchInfo { @@ -92,6 +111,8 @@ export interface CodeChunk { isPublicApi?: boolean; /** Function/method signature */ signature?: string; + /** Contextual prefix from contextual retrieval (LLM-generated) */ + contextualPrefix?: string; } export interface SearchResult extends CodeChunk { @@ -135,6 +156,8 @@ interface StoredMetadata { isPublicApi?: boolean; /** Function/method signature */ signature?: string; + /** Contextual prefix from contextual retrieval */ + contextualPrefix?: string; } interface IndexState { @@ -230,6 +253,7 @@ export class RAGSystem { private astChunker: ASTChunker; private bm25Index: BM25Index | null = null; private embeddingModelName: string; + private contextualRetrieval: ContextualRetrieval | null = null; constructor(config: RAGConfig) { this.config = { @@ -240,6 +264,7 @@ export class RAGSystem { useHybridSearch: true, // Enable hybrid search by default useReranking: false, // Disable reranking by default (slower) rrfK: 60, // Standard RRF constant + useContextualRetrieval: false, // Disable contextual retrieval by default ...config }; @@ -252,6 +277,20 @@ export class RAGSystem { extractDomainHints: this.config.extractDomainHints }); + // Initialize contextual retrieval if enabled + if (this.config.useContextualRetrieval) { + this.contextualRetrieval = new ContextualRetrieval({ + enabled: true, + useLocal: this.config.contextualLocal, + useOllama: this.config.contextualUseOllama, + apiKey: this.config.contextualApiKey, + model: this.config.contextualModel, + ollamaHost: this.config.contextualOllamaHost, + localModel: this.config.contextualLocalModel, + cacheDir: this.config.storePath, + }); + } + // Ensure cache directory exists if (!fs.existsSync(this.config.storePath)) { fs.mkdirSync(this.config.storePath, { recursive: true }); @@ -614,6 +653,29 @@ export class RAGSystem { chunksToIndex = this.prioritizeChunks(chunks, this.config.maxChunks); } + // Apply contextual retrieval if enabled + if (this.contextualRetrieval && this.config.useContextualRetrieval) { + console.log(` Enriching chunks with contextual retrieval...`); + try { + await this.contextualRetrieval.initialize(); + const enrichedChunks = await this.contextualRetrieval.enrichChunks( + chunksToIndex as ASTChunk[], + this.config.repoPath + ); + // Update chunks with contextual prefixes + chunksToIndex = enrichedChunks.map(ec => ({ + ...ec, + contextualPrefix: ec.contextualPrefix, + // Use enriched content for embedding + content: ec.enrichedContent || ec.content, + })); + const stats = this.contextualRetrieval.getStats(); + console.log(` āœ“ Contextual enrichment complete (${stats.cacheHits} cached)`); + } catch (err) { + console.warn(` āš ļø Contextual retrieval failed, continuing without: ${(err as Error).message}`); + } + } + // Generate embeddings console.log(` Generating embeddings for ${chunksToIndex.length} chunks...`); const embeddings = await this.generateEmbeddings(chunksToIndex); @@ -650,7 +712,8 @@ export class RAGSystem { domainCategories: chunk.domainCategories, domainContext: chunk.domainContext, isPublicApi: chunk.isPublicApi, - signature: chunk.signature + signature: chunk.signature, + contextualPrefix: chunk.contextualPrefix }); } @@ -738,7 +801,8 @@ export class RAGSystem { domainCategories: chunk.domainCategories, domainContext: chunk.domainContext, isPublicApi: chunk.isPublicApi, - signature: chunk.signature + signature: chunk.signature, + contextualPrefix: chunk.contextualPrefix }); }); } @@ -1658,7 +1722,8 @@ export class RAGSystem { domainCategories: chunk.domainCategories, domainContext: chunk.domainContext, isPublicApi: chunk.isPublicApi, - signature: chunk.signature + signature: chunk.signature, + contextualPrefix: chunk.contextualPrefix }); } diff --git a/src/wiki-agent.ts b/src/wiki-agent.ts index f84c26f..2ec4931 100644 --- a/src/wiki-agent.ts +++ b/src/wiki-agent.ts @@ -10,6 +10,7 @@ import { MCPConfigManager } from './mcp-config.js'; import { RAGSystem } from './rag/index.js'; import { WIKI_SYSTEM_PROMPT } from './prompts/wiki-system.js'; import { createLLMProvider, type LLMProvider, type LLMMessage, type LLMTool } from './llm/index.js'; +import { CodebaseDiscovery, generateHierarchicalIndex, generateComponentTable, type DiscoveryResult, type DiscoveredDomain } from './discovery/index.js'; export interface WikiGenerationOptions { repoUrl: string; @@ -107,6 +108,22 @@ export interface WikiGenerationOptions { /** New wiki pages that should be created */ pagesToCreate: string[]; }; + + // Contextual retrieval options + /** Enable contextual retrieval for improved chunk understanding */ + useContextualRetrieval?: boolean; + /** Use local LLM for contextual retrieval instead of Claude API */ + contextualLocal?: boolean; + /** Use Ollama server for contextual retrieval (requires contextualLocal) */ + contextualUseOllama?: boolean; + /** Anthropic API key for contextual retrieval */ + contextualApiKey?: string; + /** Model for contextual retrieval (default: claude-3-haiku) */ + contextualModel?: string; + /** Ollama host for contextual retrieval */ + contextualOllamaHost?: string; + /** Local model name for contextual retrieval */ + contextualLocalModel?: string; } export interface WikiAgentConfig { @@ -222,7 +239,15 @@ export class ArchitecturalWikiAgent { this.ragSystem = new RAGSystem({ storePath: path.join(this.outputDir, '.semanticwiki-cache'), repoPath: this.repoPath, - maxChunks: options.maxChunks // Limit chunks for large codebases + maxChunks: options.maxChunks, // Limit chunks for large codebases + // Contextual retrieval options + useContextualRetrieval: options.useContextualRetrieval, + contextualLocal: options.contextualLocal, + contextualUseOllama: options.contextualUseOllama, + contextualApiKey: options.contextualApiKey || this.config.apiKey, + contextualModel: options.contextualModel, + contextualOllamaHost: options.contextualOllamaHost || options.ollamaHost, + contextualLocalModel: options.contextualLocalModel }); await this.ragSystem.indexRepository(); yield { type: 'step', message: `Indexed ${this.ragSystem.getDocumentCount()} code chunks` }; @@ -769,7 +794,15 @@ export class ArchitecturalWikiAgent { this.ragSystem = new RAGSystem({ storePath: cachePath, repoPath: this.repoPath, - maxChunks: options.maxChunks + maxChunks: options.maxChunks, + // Contextual retrieval options + useContextualRetrieval: options.useContextualRetrieval, + contextualLocal: options.contextualLocal, + contextualUseOllama: options.contextualUseOllama, + contextualApiKey: options.contextualApiKey || this.config.apiKey, + contextualModel: options.contextualModel, + contextualOllamaHost: options.contextualOllamaHost || options.ollamaHost, + contextualLocalModel: options.contextualLocalModel }); await this.ragSystem.indexRepository(); } @@ -1143,7 +1176,15 @@ Create all ${missingPages.length} missing pages now.`; this.ragSystem = new RAGSystem({ storePath: cachePath, repoPath: this.repoPath, - maxChunks: options.maxChunks + maxChunks: options.maxChunks, + // Contextual retrieval options + useContextualRetrieval: options.useContextualRetrieval, + contextualLocal: options.contextualLocal, + contextualUseOllama: options.contextualUseOllama, + contextualApiKey: options.contextualApiKey || this.config.apiKey, + contextualModel: options.contextualModel, + contextualOllamaHost: options.contextualOllamaHost || options.ollamaHost, + contextualLocalModel: options.contextualLocalModel }); await this.ragSystem.indexRepository(); } @@ -1151,12 +1192,35 @@ Create all ${missingPages.length} missing pages now.`; const searchMode = this.ragSystem['index'] ? 'vector search' : 'keyword search'; yield { type: 'step', message: `Loaded ${this.ragSystem.getDocumentCount()} chunks (${searchMode})` }; + // Phase 2.5: Run codebase discovery for hierarchical structure + yield { type: 'phase', message: 'Discovering codebase structure', progress: 12 }; + + const discovery = new CodebaseDiscovery({ + repoPath: this.repoPath, + targetPath: options.targetPath, + verbose: options.verbose, + }); + + const discoveryResult = await discovery.discover(); + yield { type: 'step', message: `Discovered ${discoveryResult.domains.length} domains, ${discoveryResult.wikiStructure.sections.length} sections` }; + // Phase 3: Initialize local LLM provider yield { type: 'phase', message: 'Initializing local LLM', progress: 15 }; - let provider: LLMProvider; - try { - provider = await createLLMProvider({ + let provider: LLMProvider | null = null; + let pagesSinceProviderInit = 0; + const PROVIDER_RESET_THRESHOLD = 15; // Reset before hitting 20 sequence limit + + // Helper to (re)initialize provider + const initializeProvider = async (): Promise => { + if (provider && 'shutdown' in provider) { + try { + await (provider as any).shutdown(); + } catch { + // Ignore shutdown errors + } + } + const newProvider = await createLLMProvider({ fullLocal: true, useOllama: options.useOllama, ollamaHost: options.ollamaHost, @@ -1168,6 +1232,13 @@ Create all ${missingPages.length} missing pages now.`; threads: options.threads, verbose: options.verbose, }); + provider = newProvider; + pagesSinceProviderInit = 0; + return newProvider; + }; + + try { + provider = await initializeProvider(); } catch (error) { const err = error as Error; yield { type: 'error', message: `Failed to initialize local LLM: ${err.message}` }; @@ -1183,12 +1254,18 @@ Create all ${missingPages.length} missing pages now.`; console.log('[Local] Output:', this.outputDir); console.log('='.repeat(60) + '\n'); - // Phase 4: Use LLM to analyze project and determine pages to generate - yield { type: 'phase', message: 'Analyzing project with LLM', progress: 20 }; + // Phase 4: Use discovery results + LLM to plan pages + yield { type: 'phase', message: 'Planning wiki pages from discovery', progress: 20 }; - const pagesToGenerate = await this.analyzeProjectWithLLM(provider, options); + const pagesToGenerate = await this.planPagesFromDiscovery(provider, discoveryResult, options); console.log(`[Local] Identified ${pagesToGenerate.length} pages to generate\n`); + // Generate hierarchical index.md first + console.log('[Local] Generating hierarchical index.md'); + const indexContent = this.generateEnhancedIndex(discoveryResult, pagesToGenerate); + fs.writeFileSync(path.join(this.outputDir, 'index.md'), indexContent, 'utf-8'); + yield { type: 'file', message: 'index.md', detail: 'Hierarchical Index' }; + // Phase 5: Generate each page one at a time yield { type: 'phase', message: `Generating ${pagesToGenerate.length} wiki pages`, progress: 25 }; @@ -1201,8 +1278,19 @@ Create all ${missingPages.length} missing pages now.`; console.log(`\n[Local] ā”€ā”€ā”€ Page ${pagesGenerated + 1}/${pagesToGenerate.length}: ${pageSpec.title} ā”€ā”€ā”€`); + // Check if we need to reinitialize provider before hitting sequence limit + if (pagesSinceProviderInit >= PROVIDER_RESET_THRESHOLD) { + console.log(`[Local] Resetting LLM provider (${pagesSinceProviderInit} pages since last reset)...`); + try { + await initializeProvider(); + } catch (error) { + console.error(`[Local] Failed to reinitialize provider: ${(error as Error).message}`); + } + } + try { const content = await this.generateSinglePage(provider, pageSpec, pagesToGenerate, options); + pagesSinceProviderInit++; if (content) { // Write the page @@ -1222,8 +1310,36 @@ Create all ${missingPages.length} missing pages now.`; } } catch (error) { const err = error as Error; - console.error(`[Local] āœ— Failed: ${pageSpec.title} - ${err.message}`); - pagesFailed++; + // Handle sequence exhaustion - reinitialize and retry once + if (err.message.includes('No sequences left')) { + console.log(`[Local] Sequence exhausted, reinitializing provider and retrying...`); + try { + await initializeProvider(); + const content = await this.generateSinglePage(provider, pageSpec, pagesToGenerate, options); + pagesSinceProviderInit++; + + if (content) { + const pagePath = path.join(this.outputDir, pageSpec.filename); + const pageDir = path.dirname(pagePath); + if (!fs.existsSync(pageDir)) { + fs.mkdirSync(pageDir, { recursive: true }); + } + fs.writeFileSync(pagePath, content, 'utf-8'); + console.log(`[Local] āœ“ Generated (after retry): ${pageSpec.filename}`); + pagesGenerated++; + yield { type: 'file', message: pageSpec.filename, detail: pageSpec.title }; + } else { + console.log(`[Local] āš  Empty content for: ${pageSpec.title}`); + pagesFailed++; + } + } catch (retryError) { + console.error(`[Local] āœ— Failed after retry: ${pageSpec.title} - ${(retryError as Error).message}`); + pagesFailed++; + } + } else { + console.error(`[Local] āœ— Failed: ${pageSpec.title} - ${err.message}`); + pagesFailed++; + } } } @@ -1268,6 +1384,16 @@ Create all ${missingPages.length} missing pages now.`; console.log(`[Local] ā”€ā”€ā”€ Generating missing page: ${pageTitle} ā”€ā”€ā”€`); + // Check if we need to reinitialize provider + if (pagesSinceProviderInit >= PROVIDER_RESET_THRESHOLD) { + console.log(`[Local] Resetting LLM provider (${pagesSinceProviderInit} pages since last reset)...`); + try { + await initializeProvider(); + } catch (error) { + console.error(`[Local] Failed to reinitialize provider: ${(error as Error).message}`); + } + } + // Find broken links that reference this page to get context const referencingLinks = verification.brokenLinks.filter(l => l.resolvedTarget === missingFile); const linkContexts = referencingLinks.map(l => l.linkText).join(', '); @@ -1295,6 +1421,7 @@ Create all ${missingPages.length} missing pages now.`; [...pagesToGenerate, missingPageSpec], options ); + pagesSinceProviderInit++; if (content) { const pagePath = path.join(this.outputDir, missingFile); @@ -1311,8 +1438,41 @@ Create all ${missingPages.length} missing pages now.`; yield { type: 'file', message: missingFile, detail: pageTitle }; } } catch (error) { - console.error(`[Local] āœ— Failed to generate missing page: ${missingFile}`); - pagesFailed++; + const err = error as Error; + // Handle sequence exhaustion + if (err.message.includes('No sequences left')) { + console.log(`[Local] Sequence exhausted, reinitializing provider and retrying...`); + try { + await initializeProvider(); + const content = await this.generateSinglePage( + provider, + missingPageSpec, + [...pagesToGenerate, missingPageSpec], + options + ); + pagesSinceProviderInit++; + + if (content) { + const pagePath = path.join(this.outputDir, missingFile); + const pageDir = path.dirname(pagePath); + if (!fs.existsSync(pageDir)) { + fs.mkdirSync(pageDir, { recursive: true }); + } + fs.writeFileSync(pagePath, content, 'utf-8'); + console.log(`[Local] āœ“ Generated missing page (after retry): ${missingFile}`); + pagesGenerated++; + generatedFilenames.add(missingFile); + pagesToGenerate.push(missingPageSpec); + yield { type: 'file', message: missingFile, detail: pageTitle }; + } + } catch (retryError) { + console.error(`[Local] āœ— Failed to generate missing page after retry: ${missingFile}`); + pagesFailed++; + } + } else { + console.error(`[Local] āœ— Failed to generate missing page: ${missingFile}`); + pagesFailed++; + } } } } @@ -1899,6 +2059,334 @@ Return only the JSON, no markdown code blocks.`; .join(' '); } + /** + * Plan wiki pages from discovery results + * Uses the hierarchical domain structure to create a comprehensive page plan + */ + private async planPagesFromDiscovery( + provider: LLMProvider, + discoveryResult: DiscoveryResult, + options: WikiGenerationOptions + ): Promise> { + const pages: Array<{ + title: string; + filename: string; + type: 'overview' | 'component' | 'api' | 'guide' | 'module' | 'config' | 'relationship'; + context: string; + sourceFiles: string[]; + section?: string; + domain?: string; + }> = []; + + // 1. Add architecture overview page + pages.push({ + title: 'Architecture Overview', + filename: 'architecture.md', + type: 'overview', + context: `High-level architecture of ${discoveryResult.project.name}. Project type: ${discoveryResult.project.type}. Technologies: ${discoveryResult.project.technologies.join(', ')}. Cover system components, data flow, and integration points.`, + sourceFiles: discoveryResult.domains.flatMap(d => d.files.slice(0, 3)), + section: 'Architecture', + }); + + // 2. Add getting started page + pages.push({ + title: 'Getting Started', + filename: 'getting-started.md', + type: 'guide', + context: `Setup and development guide for ${discoveryResult.project.name}. Include installation, configuration, and basic usage.`, + sourceFiles: ['README.md', 'package.json', '.env.example'].filter(f => + discoveryResult.domains.some(d => d.files.includes(f)) + ), + section: 'Guides', + }); + + // 3. Add pages for each section from discovery + // Create a map of domain IDs to domains for quick lookup + const domainMap = new Map(discoveryResult.domains.map(d => [d.id, d])); + + for (const section of discoveryResult.wikiStructure.sections) { + // Find all domains in this section for component tables + const sectionDomains = discoveryResult.domains.filter(d => + this.categoryToSection(d.category) === section.title + ); + + // Generate combined component tables for section overview + const sectionComponentTables = sectionDomains + .map(d => generateComponentTable(d)) + .filter(t => t.length > 0) + .join('\n'); + + // Add section overview if multiple pages + if (section.pages.length > 1) { + let overviewContext = section.description; + if (sectionComponentTables) { + overviewContext += `\n\n## Component Reference\n\n${sectionComponentTables}`; + } + + pages.push({ + title: section.title, + filename: `${section.id}.md`, + type: 'overview', + context: overviewContext, + sourceFiles: section.pages.flatMap(p => p.sourcePaths).slice(0, 10), + section: section.title, + }); + } + + // Add pages for each domain in this section + for (const page of section.pages) { + // Skip if it's the same as section overview + if (page.slug === `${section.id}.md`) continue; + + // Extract domain ID from page slug (e.g., "domain-id.md" -> "domain-id") + const domainId = page.slug.replace('.md', '').replace(/-\w+$/, ''); + const domain = domainMap.get(domainId); + + // Generate component table for this domain if available + let pageContext = page.description; + if (domain) { + const componentTable = generateComponentTable(domain); + if (componentTable) { + pageContext += `\n\n${componentTable}`; + } + } + + pages.push({ + title: page.title, + filename: page.slug, + type: page.pageType === 'overview' ? 'module' : page.pageType === 'feature' ? 'component' : 'module', + context: pageContext, + sourceFiles: page.sourcePaths, + section: section.title, + }); + } + } + + // 4. Add relationship pages if significant cross-domain interactions + if (discoveryResult.relationships.length > 5) { + pages.push({ + title: 'Data Flow', + filename: 'data-flow.md', + type: 'relationship', + context: 'How data moves through the system. Include data transformations, storage patterns, and integration points.', + sourceFiles: discoveryResult.domains + .filter(d => d.category === 'data-layer' || d.category === 'core-application') + .flatMap(d => d.files.slice(0, 5)), + section: 'System Relationships', + }); + + pages.push({ + title: 'Integration Points', + filename: 'integration-points.md', + type: 'relationship', + context: 'How modules connect and communicate. Include APIs, data contracts, and dependencies.', + sourceFiles: discoveryResult.domains + .filter(d => d.category === 'integration' || d.relatedDomains.length > 2) + .flatMap(d => d.files.slice(0, 5)), + section: 'System Relationships', + }); + } + + // 5. Ensure minimum page count by adding domain-specific pages + const minPages = Math.max(8, discoveryResult.domains.length); + if (pages.length < minPages) { + for (const domain of discoveryResult.domains) { + const existingPage = pages.find(p => p.filename === `${domain.id}.md`); + if (!existingPage && pages.length < minPages + 5) { + // Generate component table for this domain + const componentTable = generateComponentTable(domain); + let domainContext = `${domain.description}. ${domain.businessPurpose || ''}`; + if (componentTable) { + domainContext += `\n\n${componentTable}`; + } + + pages.push({ + title: domain.name, + filename: `${domain.id}.md`, + type: 'module', + context: domainContext, + sourceFiles: domain.files.slice(0, 10), + section: this.categoryToSection(domain.category), + domain: domain.id, + }); + } + } + } + + console.log(`[Local] Planned ${pages.length} pages from discovery (${discoveryResult.domains.length} domains)`); + return pages; + } + + /** + * Map domain category to section name + */ + private categoryToSection(category: string): string { + const sectionMap: Record = { + 'core-application': 'Core Application', + 'presentation': 'User Interface', + 'data-layer': 'Data Layer', + 'batch-processing': 'Batch Processing', + 'integration': 'Integration', + 'configuration': 'Configuration', + 'infrastructure': 'Infrastructure', + 'documentation': 'Documentation', + 'testing': 'Testing', + }; + return sectionMap[category] || 'Components'; + } + + /** + * Generate enhanced hierarchical index from discovery results + */ + private generateEnhancedIndex( + discoveryResult: DiscoveryResult, + pages: Array<{ title: string; filename: string; type: string; context: string; section?: string; domain?: string }> + ): string { + const lines: string[] = []; + + // Header with project info + lines.push(`# ${discoveryResult.project.name}`); + lines.push(''); + lines.push(discoveryResult.project.description); + lines.push(''); + + // Technology badges + if (discoveryResult.project.technologies.length > 0) { + lines.push('**Technologies:** ' + discoveryResult.project.technologies.slice(0, 8).join(' ā€¢ ')); + lines.push(''); + } + + // Project type indicator + lines.push(`**Project Type:** ${this.formatProjectType(discoveryResult.project.type)}`); + lines.push(''); + lines.push('---'); + lines.push(''); + + // Overview section + lines.push('## Overview'); + lines.push(''); + lines.push('> High-level documentation and getting started guides.'); + lines.push(''); + const overviewPages = pages.filter(p => p.type === 'overview' || p.type === 'guide'); + for (const page of overviewPages.slice(0, 5)) { + lines.push(`- šŸ“š [${page.title}](./${page.filename}) ā€“ ${this.truncateContext(page.context)}`); + } + lines.push(''); + + // Group remaining pages by section + const sectionPages = new Map(); + for (const page of pages) { + if (page.type === 'overview' || page.type === 'guide') continue; + const section = page.section || 'Components'; + const existing = sectionPages.get(section) || []; + existing.push(page); + sectionPages.set(section, existing); + } + + // Output each section + const sectionOrder = [ + 'Core Application', + 'User Interface', + 'Data Layer', + 'Batch Processing', + 'Integration', + 'System Relationships', + 'Configuration', + 'Infrastructure', + 'Components', + ]; + + for (const sectionName of sectionOrder) { + const sectionPageList = sectionPages.get(sectionName); + if (!sectionPageList || sectionPageList.length === 0) continue; + + lines.push(`## ${sectionName}`); + lines.push(''); + + // Find matching section in discovery for description + const discoveredSection = discoveryResult.wikiStructure.sections.find(s => s.title === sectionName); + if (discoveredSection) { + lines.push(`> ${discoveredSection.description}`); + lines.push(''); + } + + for (const page of sectionPageList) { + const icon = this.getPageIcon(page.type); + lines.push(`- ${icon} [${page.title}](./${page.filename}) ā€“ ${this.truncateContext(page.context)}`); + } + lines.push(''); + } + + // Quick reference table + lines.push('---'); + lines.push(''); + lines.push('## Quick Reference'); + lines.push(''); + lines.push('| Domain | Description | Files | Category |'); + lines.push('|--------|-------------|-------|----------|'); + + for (const domain of discoveryResult.domains.slice(0, 12)) { + const pageLink = pages.find(p => p.domain === domain.id || p.filename === `${domain.id}.md`); + const nameCell = pageLink ? `[${domain.name}](./${pageLink.filename})` : domain.name; + lines.push(`| ${nameCell} | ${this.truncateContext(domain.description, 40)} | ${domain.files.length} | ${domain.category} |`); + } + + lines.push(''); + lines.push('---'); + lines.push(''); + lines.push(`*Generated by SemanticWiki with hierarchical discovery ā€¢ ${discoveryResult.stats.totalFiles} files analyzed ā€¢ ${discoveryResult.stats.totalDomains} domains discovered*`); + + return lines.join('\n'); + } + + /** + * Format project type for display + */ + private formatProjectType(type: string): string { + const typeMap: Record = { + 'mainframe-cobol': 'šŸ–„ļø Mainframe COBOL Application', + 'web-application': 'šŸŒ Web Application', + 'api-service': 'šŸ”Œ API Service', + 'cli-tool': 'āŒØļø CLI Tool', + 'library': 'šŸ“¦ Library', + 'monorepo': 'šŸ“ Monorepo', + 'unknown': 'šŸ“„ Software Project', + }; + return typeMap[type] || type; + } + + /** + * Get icon for page type + */ + private getPageIcon(type: string): string { + const iconMap: Record = { + 'overview': 'šŸ“š', + 'component': 'āš™ļø', + 'module': 'šŸ“¦', + 'guide': 'šŸ“', + 'config': 'šŸ”§', + 'api': 'šŸ”Œ', + 'relationship': 'šŸ”—', + }; + return iconMap[type] || 'šŸ“„'; + } + + /** + * Truncate context string for display + */ + private truncateContext(context: string, maxLength: number = 60): string { + if (context.length <= maxLength) return context; + return context.slice(0, maxLength - 3) + '...'; + } + /** * Gather all project files with metadata for LLM analysis */ diff --git a/tests/contextual-retrieval.test.ts b/tests/contextual-retrieval.test.ts new file mode 100644 index 0000000..866f789 --- /dev/null +++ b/tests/contextual-retrieval.test.ts @@ -0,0 +1,553 @@ +/** + * Tests for Contextual Retrieval Module + * + * Tests the contextual retrieval functionality that enriches code chunks + * with LLM-generated context for improved semantic search. + * + * Reference: https://www.anthropic.com/news/contextual-retrieval + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; + +// Mock the Anthropic SDK +vi.mock('@anthropic-ai/sdk', () => ({ + default: vi.fn().mockImplementation(() => ({ + messages: { + create: vi.fn().mockResolvedValue({ + content: [{ + type: 'text', + text: 'This code is from auth.ts. It handles user authentication using JWT tokens.' + }] + }) + } + })) +})); + +// Mock the LLM provider for local mode +vi.mock('../src/llm/index.js', () => ({ + createLLMProvider: vi.fn().mockResolvedValue({ + initialize: vi.fn().mockResolvedValue(undefined), + chat: vi.fn().mockResolvedValue({ + content: 'This is a function that validates user credentials.', + toolCalls: [], + stopReason: 'end_turn', + usage: { inputTokens: 100, outputTokens: 50 } + }), + shutdown: vi.fn().mockResolvedValue(undefined), + getModelInfo: vi.fn().mockReturnValue({ + name: 'gpt-oss-21b', + contextLength: 32768, + supportsTools: true, + supportsStreaming: true, + isLocal: true + }) + }) +})); + +// Import after mocks are set up +import { + ContextualRetrieval, + ContextualRetrievalConfig, + ContextualChunk, + estimateContextualRetrievalCost +} from '../src/rag/contextual-retrieval.js'; +import type { ASTChunk } from '../src/ast-chunker.js'; + +describe('ContextualRetrieval', () => { + const testCacheDir = '/tmp/test-contextual-cache'; + + beforeEach(() => { + // Clean up cache directory + if (fs.existsSync(testCacheDir)) { + fs.rmSync(testCacheDir, { recursive: true, force: true }); + } + fs.mkdirSync(testCacheDir, { recursive: true }); + }); + + afterEach(() => { + if (fs.existsSync(testCacheDir)) { + fs.rmSync(testCacheDir, { recursive: true, force: true }); + } + }); + + describe('Configuration', () => { + it('should use default values when not provided', () => { + const config: ContextualRetrievalConfig = { + enabled: true + }; + + const retrieval = new ContextualRetrieval(config); + const stats = retrieval.getStats(); + + expect(stats.cacheSize).toBe(0); + expect(stats.filesProcessed).toBe(0); + }); + + it('should respect enabled flag', async () => { + const config: ContextualRetrievalConfig = { + enabled: false + }; + + const retrieval = new ContextualRetrieval(config); + const mockChunks: ASTChunk[] = [{ + id: 'test-1', + filePath: 'src/test.ts', + startLine: 1, + endLine: 10, + content: 'function test() {}', + language: 'typescript' + }]; + + const result = await retrieval.enrichChunks(mockChunks, '/test/repo'); + + // When disabled, should return chunks with empty context + expect(result[0].contextualPrefix).toBe(''); + expect(result[0].enrichedContent).toBe(mockChunks[0].content); + }); + + it('should use Claude API by default when not using local', () => { + const config: ContextualRetrievalConfig = { + enabled: true, + useLocal: false, + apiKey: 'test-api-key' + }; + + const retrieval = new ContextualRetrieval(config); + // The retrieval should be configured for Claude API + expect(retrieval).toBeDefined(); + }); + + it('should configure for Ollama when useLocal and useOllama are true', () => { + const config: ContextualRetrievalConfig = { + enabled: true, + useLocal: true, + useOllama: true, + ollamaHost: 'http://localhost:11434', + localModel: 'qwen2.5-coder:7b' + }; + + const retrieval = new ContextualRetrieval(config); + expect(retrieval).toBeDefined(); + }); + + it('should configure for bundled local when useLocal is true without useOllama', () => { + const config: ContextualRetrievalConfig = { + enabled: true, + useLocal: true, + useOllama: false + }; + + const retrieval = new ContextualRetrieval(config); + expect(retrieval).toBeDefined(); + }); + }); + + describe('Fallback Context Generation', () => { + it('should generate fallback context from AST metadata', async () => { + const config: ContextualRetrievalConfig = { + enabled: true, + useLocal: true, + cacheDir: testCacheDir + }; + + const retrieval = new ContextualRetrieval(config); + + // Access private method via prototype (for testing) + const generateFallback = (retrieval as any).generateFallbackContext.bind(retrieval); + + const chunk: ASTChunk = { + id: 'test-1', + filePath: 'src/auth/login.ts', + startLine: 10, + endLine: 25, + content: 'async function validateCredentials() {}', + language: 'typescript', + chunkType: 'function', + name: 'validateCredentials', + domainHints: [ + { category: 'authentication', confidence: 0.9 } + ] + }; + + const context = generateFallback(chunk); + + expect(context).toContain('login.ts'); + expect(context).toContain('validateCredentials'); + expect(context).toContain('function'); + expect(context).toContain('authentication'); + }); + + it('should handle chunks without optional metadata', () => { + const config: ContextualRetrievalConfig = { + enabled: true, + useLocal: true + }; + + const retrieval = new ContextualRetrieval(config); + const generateFallback = (retrieval as any).generateFallbackContext.bind(retrieval); + + const chunk: ASTChunk = { + id: 'test-2', + filePath: 'src/utils/helpers.ts', + startLine: 1, + endLine: 5, + content: 'const x = 1;', + language: 'typescript' + }; + + const context = generateFallback(chunk); + + expect(context).toContain('helpers.ts'); + expect(context).not.toContain('undefined'); + }); + + it('should include parent name when available', () => { + const config: ContextualRetrievalConfig = { + enabled: true, + useLocal: true + }; + + const retrieval = new ContextualRetrieval(config); + const generateFallback = (retrieval as any).generateFallbackContext.bind(retrieval); + + const chunk: ASTChunk = { + id: 'test-3', + filePath: 'src/services/UserService.ts', + startLine: 20, + endLine: 30, + content: 'async getUserById(id: string) {}', + language: 'typescript', + chunkType: 'method', + name: 'getUserById', + parentName: 'UserService' + }; + + const context = generateFallback(chunk); + + expect(context).toContain('UserService'); + expect(context).toContain('getUserById'); + }); + }); + + describe('Content Truncation', () => { + it('should truncate long content at natural boundaries', () => { + const config: ContextualRetrievalConfig = { + enabled: true + }; + + const retrieval = new ContextualRetrieval(config); + const truncate = (retrieval as any).truncateContent.bind(retrieval); + + // Use a much longer content to properly test truncation + const longContent = 'Line 1\nLine 2\nLine 3\nLine 4\nLine 5\nLine 6\nLine 7\nLine 8\nLine 9\nLine 10'; + const truncated = truncate(longContent, 30); + + expect(truncated).toContain('[truncated]'); + // The base content before marker should be truncated + expect(truncated.indexOf('[truncated]')).toBeGreaterThan(0); + }); + + it('should not truncate content shorter than limit', () => { + const config: ContextualRetrievalConfig = { + enabled: true + }; + + const retrieval = new ContextualRetrieval(config); + const truncate = (retrieval as any).truncateContent.bind(retrieval); + + const shortContent = 'Hello World'; + const result = truncate(shortContent, 100); + + expect(result).toBe(shortContent); + expect(result).not.toContain('[truncated]'); + }); + }); + + describe('Cache Management', () => { + it('should save and load cache', () => { + const config: ContextualRetrievalConfig = { + enabled: true, + cacheDir: testCacheDir + }; + + // Create retrieval and manually add to cache + const retrieval = new ContextualRetrieval(config); + const contextCache = (retrieval as any).contextCache as Map; + contextCache.set('test-key:100', 'Test context for chunk'); + + // Save cache + (retrieval as any).saveCache(); + + // Create new retrieval instance - should load cache + const retrieval2 = new ContextualRetrieval(config); + const loadedCache = (retrieval2 as any).contextCache as Map; + + expect(loadedCache.get('test-key:100')).toBe('Test context for chunk'); + }); + + it('should use cache key based on chunk id and content length', () => { + const chunk: ASTChunk = { + id: 'src/auth.ts:10-20', + filePath: 'src/auth.ts', + startLine: 10, + endLine: 20, + content: 'function authenticate() { /* ... */ }', + language: 'typescript' + }; + + const cacheKey = `${chunk.id}:${chunk.content.length}`; + expect(cacheKey).toBe('src/auth.ts:10-20:37'); + }); + }); + + describe('Statistics', () => { + it('should track cache statistics', () => { + const config: ContextualRetrievalConfig = { + enabled: true, + cacheDir: testCacheDir + }; + + const retrieval = new ContextualRetrieval(config); + + // Add some entries to the internal caches + const contextCache = (retrieval as any).contextCache as Map; + const fileCache = (retrieval as any).fileContentCache as Map; + + contextCache.set('key1', 'context1'); + contextCache.set('key2', 'context2'); + fileCache.set('file1.ts', 'content1'); + + const stats = retrieval.getStats(); + + expect(stats.cacheSize).toBe(2); + expect(stats.filesProcessed).toBe(1); + }); + }); +}); + +describe('Cost Estimation', () => { + it('should estimate cost for contextual retrieval', () => { + const estimate = estimateContextualRetrievalCost(1000, 5000); + + expect(estimate.inputTokens).toBeGreaterThan(0); + expect(estimate.outputTokens).toBe(100000); // 100 tokens per chunk + expect(estimate.estimatedCost).toBeGreaterThan(0); + expect(estimate.withCaching).toBeLessThan(estimate.estimatedCost); + }); + + it('should show significant savings with caching', () => { + const estimate = estimateContextualRetrievalCost(10000); + + // Caching should reduce cost by ~90% on input side + const savingsRatio = estimate.withCaching / estimate.estimatedCost; + expect(savingsRatio).toBeLessThan(0.5); // At least 50% savings + }); + + it('should scale linearly with chunk count', () => { + const estimate1 = estimateContextualRetrievalCost(1000); + const estimate2 = estimateContextualRetrievalCost(2000); + + // Cost should roughly double + expect(estimate2.estimatedCost).toBeCloseTo(estimate1.estimatedCost * 2, 1); + }); + + it('should account for file size in token estimation', () => { + const smallFiles = estimateContextualRetrievalCost(1000, 2000); + const largeFiles = estimateContextualRetrievalCost(1000, 10000); + + expect(largeFiles.inputTokens).toBeGreaterThan(smallFiles.inputTokens); + }); +}); + +describe('Enriched Chunks', () => { + it('should combine context prefix with original content', () => { + const contextPrefix = 'This code is from auth.ts and handles user login.'; + const originalContent = 'function login(user, pass) { /* ... */ }'; + + const enrichedContent = `${contextPrefix}\n\n${originalContent}`; + + expect(enrichedContent).toContain(contextPrefix); + expect(enrichedContent).toContain(originalContent); + expect(enrichedContent.indexOf(contextPrefix)).toBeLessThan( + enrichedContent.indexOf(originalContent) + ); + }); + + it('should preserve original chunk properties', () => { + const originalChunk: ASTChunk = { + id: 'test-1', + filePath: 'src/test.ts', + startLine: 1, + endLine: 10, + content: 'function test() {}', + language: 'typescript', + chunkType: 'function', + name: 'test' + }; + + const enrichedChunk: ContextualChunk = { + ...originalChunk, + contextualPrefix: 'Test context', + enrichedContent: 'Test context\n\nfunction test() {}' + }; + + expect(enrichedChunk.id).toBe(originalChunk.id); + expect(enrichedChunk.filePath).toBe(originalChunk.filePath); + expect(enrichedChunk.chunkType).toBe(originalChunk.chunkType); + expect(enrichedChunk.name).toBe(originalChunk.name); + }); +}); + +describe('Context Prompt Template', () => { + const EXPECTED_PROMPT_STRUCTURE = ` +{WHOLE_DOCUMENT} + + +Here is the chunk we want to situate within the whole document: + +{CHUNK_CONTENT} +`; + + it('should use XML-style document and chunk markers', () => { + expect(EXPECTED_PROMPT_STRUCTURE).toContain(''); + expect(EXPECTED_PROMPT_STRUCTURE).toContain(''); + expect(EXPECTED_PROMPT_STRUCTURE).toContain(''); + expect(EXPECTED_PROMPT_STRUCTURE).toContain(''); + }); + + it('should have placeholders for document and chunk content', () => { + expect(EXPECTED_PROMPT_STRUCTURE).toContain('{WHOLE_DOCUMENT}'); + expect(EXPECTED_PROMPT_STRUCTURE).toContain('{CHUNK_CONTENT}'); + }); +}); + +describe('Mode Selection', () => { + interface ModeConfig { + useLocal: boolean; + useOllama: boolean; + } + + function determineMode(config: ModeConfig): 'claude-api' | 'ollama' | 'bundled-local' { + if (!config.useLocal) { + return 'claude-api'; + } + if (config.useOllama) { + return 'ollama'; + } + return 'bundled-local'; + } + + it('should use Claude API when useLocal is false', () => { + expect(determineMode({ useLocal: false, useOllama: false })).toBe('claude-api'); + expect(determineMode({ useLocal: false, useOllama: true })).toBe('claude-api'); + }); + + it('should use Ollama when both useLocal and useOllama are true', () => { + expect(determineMode({ useLocal: true, useOllama: true })).toBe('ollama'); + }); + + it('should use bundled local when useLocal is true and useOllama is false', () => { + expect(determineMode({ useLocal: true, useOllama: false })).toBe('bundled-local'); + }); +}); + +describe('Concurrency Settings', () => { + it('should use higher concurrency for API mode', () => { + const apiConcurrency = 5; + const localConcurrency = 1; + + expect(apiConcurrency).toBeGreaterThan(localConcurrency); + }); + + it('should process sequentially for local mode to manage memory', () => { + const config: ContextualRetrievalConfig = { + enabled: true, + useLocal: true, + useOllama: false + }; + + // When useLocal is true, concurrency should default to 1 + const expectedConcurrency = config.useLocal ? 1 : 5; + expect(expectedConcurrency).toBe(1); + }); +}); + +describe('Error Handling', () => { + it('should fall back to AST-based context on LLM failure', () => { + // Simulate what happens when LLM call fails + const chunk: ASTChunk = { + id: 'test-1', + filePath: 'src/auth/jwt.ts', + startLine: 1, + endLine: 20, + content: 'export function verifyToken(token: string) {}', + language: 'typescript', + chunkType: 'function', + name: 'verifyToken', + domainHints: [ + { category: 'authentication', confidence: 0.95 } + ] + }; + + // Fallback context should include available metadata + const fallbackParts: string[] = []; + fallbackParts.push(`This code is from ${path.basename(chunk.filePath)}.`); + if (chunk.chunkType && chunk.name) { + fallbackParts.push(`It defines ${chunk.name}, a ${chunk.chunkType}.`); + } + + const fallbackContext = fallbackParts.join(' '); + + expect(fallbackContext).toContain('jwt.ts'); + expect(fallbackContext).toContain('verifyToken'); + expect(fallbackContext).toContain('function'); + }); + + it('should continue processing other chunks when one fails', async () => { + // This tests the principle that failures are isolated + const chunks: ASTChunk[] = [ + { id: '1', filePath: 'a.ts', startLine: 1, endLine: 5, content: 'code1', language: 'ts' }, + { id: '2', filePath: 'b.ts', startLine: 1, endLine: 5, content: 'code2', language: 'ts' }, + { id: '3', filePath: 'c.ts', startLine: 1, endLine: 5, content: 'code3', language: 'ts' } + ]; + + // Even if one chunk fails, the others should be processed + // The implementation catches errors per-chunk and uses fallback + expect(chunks.length).toBe(3); + }); +}); + +describe('Integration with RAG System', () => { + it('should produce enriched content suitable for embeddings', () => { + const contextPrefix = 'This is the authentication module that handles JWT token verification.'; + const originalCode = ` +export function verifyToken(token: string): boolean { + return jwt.verify(token, secret); +}`; + + const enrichedContent = `${contextPrefix}\n\n${originalCode}`; + + // The enriched content should contain: + // 1. Business context (what the code does) + // 2. The original code + expect(enrichedContent).toContain('authentication'); + expect(enrichedContent).toContain('JWT'); + expect(enrichedContent).toContain('verifyToken'); + expect(enrichedContent).toContain('jwt.verify'); + }); + + it('should improve retrieval by adding semantic context', () => { + // Original code might not match query "user login" + const originalCode = 'function authenticate(creds) { return true; }'; + + // With contextual prefix, it can now match + const enrichedCode = 'This function handles user login by validating credentials.\n\n' + originalCode; + + // The enriched version now contains relevant terms + expect(enrichedCode.toLowerCase()).toContain('user'); + expect(enrichedCode.toLowerCase()).toContain('login'); + expect(enrichedCode.toLowerCase()).toContain('credentials'); + }); +});