diff --git a/package-lock.json b/package-lock.json index 31f4c45..7201b8e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "packages/*" ], "devDependencies": { + "@types/node": "^24.0.0", "tsup": "^8.5.1", "vitest": "^4.1.2" } @@ -1209,16 +1210,6 @@ "tslib": "^2.4.0" } }, - "node_modules/@types/better-sqlite3": { - "version": "7.6.13", - "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz", - "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -1245,13 +1236,13 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "25.5.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.0.tgz", - "integrity": "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==", + "version": "24.12.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.2.tgz", + "integrity": "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==", "dev": true, "license": "MIT", "dependencies": { - "undici-types": "~7.18.0" + "undici-types": "~7.16.0" } }, "node_modules/@vitest/expect": { @@ -1397,81 +1388,6 @@ "node": ">=12" } }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/better-sqlite3": { - "version": "11.10.0", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz", - "integrity": "sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==", - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "bindings": "^1.5.0", - "prebuild-install": "^7.1.1" - } - }, - "node_modules/bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "license": "MIT", - "dependencies": { - "file-uri-to-path": "1.0.0" - } - }, - "node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "license": "MIT", - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, "node_modules/bundle-require": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/bundle-require/-/bundle-require-5.1.0.tgz", @@ -1523,12 +1439,6 @@ "url": "https://paulmillr.com/funding/" } }, - "node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "license": "ISC" - }, "node_modules/commander": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", @@ -1581,48 +1491,16 @@ } } }, - "node_modules/decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "license": "MIT", - "dependencies": { - "mimic-response": "^3.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "license": "MIT", - "engines": { - "node": ">=4.0.0" - } - }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=8" } }, - "node_modules/end-of-stream": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", - "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "license": "MIT", - "dependencies": { - "once": "^1.4.0" - } - }, "node_modules/es-module-lexer": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.0.0.tgz", @@ -1682,15 +1560,6 @@ "@types/estree": "^1.0.0" } }, - "node_modules/expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "license": "(MIT OR WTFPL)", - "engines": { - "node": ">=6" - } - }, "node_modules/expect-type": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", @@ -1719,12 +1588,6 @@ } } }, - "node_modules/file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "license": "MIT" - }, "node_modules/fix-dts-default-cjs-exports": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fix-dts-default-cjs-exports/-/fix-dts-default-cjs-exports-1.0.1.tgz", @@ -1737,12 +1600,6 @@ "rollup": "^4.34.8" } }, - "node_modules/fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "license": "MIT" - }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -1771,44 +1628,6 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, - "node_modules/github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "license": "MIT" - }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, - "node_modules/ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "license": "ISC" - }, "node_modules/joycon": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", @@ -2124,33 +1943,6 @@ "resolved": "packages/openclaw-plugin", "link": true }, - "node_modules/mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "license": "MIT" - }, "node_modules/mlly": { "version": "1.8.2", "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.2.tgz", @@ -2202,24 +1994,6 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, - "node_modules/napi-build-utils": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", - "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", - "license": "MIT" - }, - "node_modules/node-abi": { - "version": "3.89.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz", - "integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==", - "license": "MIT", - "dependencies": { - "semver": "^7.3.5" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -2241,15 +2015,6 @@ ], "license": "MIT" }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -2371,72 +2136,6 @@ } } }, - "node_modules/prebuild-install": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", - "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", - "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", - "license": "MIT", - "dependencies": { - "detect-libc": "^2.0.0", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.3", - "mkdirp-classic": "^0.5.3", - "napi-build-utils": "^2.0.0", - "node-abi": "^3.3.0", - "pump": "^3.0.0", - "rc": "^1.2.7", - "simple-get": "^4.0.0", - "tar-fs": "^2.0.0", - "tunnel-agent": "^0.6.0" - }, - "bin": { - "prebuild-install": "bin.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/pump": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", - "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", - "license": "MIT", - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "node_modules/rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", - "dependencies": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "bin": { - "rc": "cli.js" - } - }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/readdirp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", @@ -2549,38 +2248,6 @@ "fsevents": "~2.3.2" } }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -2588,51 +2255,6 @@ "dev": true, "license": "ISC" }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/simple-get": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", - "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "decompress-response": "^6.0.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, "node_modules/source-map": { "version": "0.7.6", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", @@ -2667,24 +2289,6 @@ "dev": true, "license": "MIT" }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, - "node_modules/strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/sucrase": { "version": "3.35.1", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz", @@ -2708,34 +2312,6 @@ "node": ">=16 || 14 >=14.17" } }, - "node_modules/tar-fs": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", - "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", - "license": "MIT", - "dependencies": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.1.4" - } - }, - "node_modules/tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "license": "MIT", - "dependencies": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/thenify": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", @@ -2908,18 +2484,6 @@ "fsevents": "~2.3.3" } }, - "node_modules/tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "license": "Apache-2.0", - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -2942,18 +2506,12 @@ "license": "MIT" }, "node_modules/undici-types": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", - "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "dev": true, "license": "MIT" }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "license": "MIT" - }, "node_modules/vite": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.3.tgz", @@ -3131,12 +2689,6 @@ "node": ">=8" } }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC" - }, "packages/daemon": { "name": "@memrok/daemon", "version": "0.5.1", @@ -3170,7 +2722,6 @@ "version": "0.5.1", "license": "MIT", "dependencies": { - "better-sqlite3": "^11.9.1", "chokidar": "^4.0.3" }, "devDependencies": { @@ -3194,10 +2745,6 @@ "name": "@memrok/store", "version": "0.5.1", "dependencies": { - "better-sqlite3": "^11.9.1" - }, - "devDependencies": { - "@types/better-sqlite3": "^7.6.13", "tsx": "^4.19.4", "typescript": "^5.8.2" } diff --git a/package.json b/package.json index 284e1cb..a52a7aa 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "url": "https://github.com/memrok-com/memrok.git" }, "devDependencies": { + "@types/node": "^24.0.0", "tsup": "^8.5.1", "vitest": "^4.1.2" } diff --git a/packages/daemon/src/watcher.ts b/packages/daemon/src/watcher.ts index c7bbe6e..1c2641b 100644 --- a/packages/daemon/src/watcher.ts +++ b/packages/daemon/src/watcher.ts @@ -1,5 +1,6 @@ import { watch, type FSWatcher } from 'chokidar'; import { readFileSync, writeFileSync, statSync, openSync, readSync, closeSync } from 'node:fs'; +import { homedir } from 'node:os'; import { resolve } from 'node:path'; import { EventEmitter } from 'node:events'; import type { WatcherConfig, CursorState } from './types.js'; @@ -22,7 +23,10 @@ export class TranscriptWatcher extends EventEmitter { super(); this.config = config; this.debounceMs = config.debounceMs ?? DEFAULT_DEBOUNCE_MS; - this.cursorPath = cursorPath ?? resolve(process.cwd(), '.memrok-cursors.json'); + const baseDir = process.env.OPENCLAW_DATA_DIR + || process.env.OPENCLAW_STATE_DIR + || resolve(homedir(), '.openclaw'); + this.cursorPath = cursorPath ?? resolve(baseDir, '.memrok-cursors.json'); this.loadCursors(); } @@ -43,29 +47,53 @@ export class TranscriptWatcher extends EventEmitter { return { ...this.cursors }; } - readNewContent(filePath: string): string | null { - const offset = this.cursors[filePath] ?? 0; - let size: number; + setCursor(filePath: string, offset: number): void { + this.cursors[filePath] = Math.max(0, offset); + } + + getFileSize(filePath: string): number | null { try { - size = statSync(filePath).size; + return statSync(filePath).size; } catch { return null; } + } - if (size <= offset) return null; + readContentFromOffset(filePath: string, offset: number): { content: string | null; nextOffset: number } | null { + const size = this.getFileSize(filePath); + if (size === null) { + return null; + } + + if (size <= offset) { + return { content: null, nextOffset: offset }; + } const fd = openSync(filePath, 'r'); try { - const length = size - offset; + const nextOffset = size; + const length = nextOffset - offset; const buf = Buffer.alloc(length); readSync(fd, buf, 0, length, offset); - this.cursors[filePath] = size; - return buf.toString('utf-8'); + return { + content: buf.toString('utf-8'), + nextOffset, + }; } finally { closeSync(fd); } } + readNewContent(filePath: string): string | null { + const offset = this.cursors[filePath] ?? 0; + const result = this.readContentFromOffset(filePath, offset); + if (!result?.content) { + return null; + } + this.cursors[filePath] = result.nextOffset; + return result.content; + } + start(): void { if (this.fsWatcher) return; diff --git a/packages/openclaw-plugin/README.md b/packages/openclaw-plugin/README.md index d0f373d..6f9c60f 100644 --- a/packages/openclaw-plugin/README.md +++ b/packages/openclaw-plugin/README.md @@ -63,12 +63,6 @@ Then activate Memrok as your context engine. Add to your `openclaw.json` (or use "memrok": { "enabled": true, "config": { - "scribeProvider": "openai", - "scribeModel": "gpt-5-mini", - "reflection": { - "provider": "openai", - "model": "gpt-5" - }, "bootstrap": { "enabled": false } @@ -79,45 +73,41 @@ Then activate Memrok as your context engine. Add to your `openclaw.json` (or use } ``` -Restart OpenClaw. Memrok watches session transcripts automatically and begins curating after the first idle window. - -Set both transcript and reflection provider/model explicitly in the plugin config if you do not want Memrok falling back to its built-in defaults. -Bootstrap is now **opt-in**. Enable it only if you explicitly want Memrok to seed itself from existing Markdown memory files. +Config lives under `plugins.entries.memrok.config`. Install puts the plugin in place, but your running gateway still needs a restart before it begins using the new build. -## Inspection & Evaluation +Restart OpenClaw. Memrok watches session transcripts automatically and begins curating after the first idle window. -Memrok ships with local inspection scripts so you can evaluate injected headers without writing back into Memrok state. +By default, Memrok uses the OpenClaw provider/model configuration already active in your runtime. Set transcript or reflection provider/model explicitly only when you want Memrok to diverge from the OpenClaw defaults. +Bootstrap is **opt-in**. Enable it only if you explicitly want Memrok to seed itself from existing Markdown memory files. When enabled, Memrok scans `MEMORY.md` and `memory/` across configured OpenClaw agents, not just the current workspace. +By default, Memrok stores its database and status files under the active OpenClaw state directory, typically `~/.openclaw/plugins/memrok/`. -Examples: +## Commands -```sh -node scripts/eval-sessions.mjs --all-sessions --dry-run --json -node scripts/eval-sessions.mjs --recent-sessions 10 --dry-run --headers -node scripts/eval-sessions.mjs --session-id --dry-run --headers -``` +Memrok also registers a `/memrok` command for operator tasks: -Notes: -- `--dry-run` / `--no-persist` prevents working-set snapshot writes during probing. -- `--json` includes the full rendered header as `headerText`. -- `--headers` is for human-readable terminal/file output. -- Optional filters such as `--topic`, `--channel`, `--provider`, and `--label` narrow session selection without changing the session-first model. +- `/memrok status` shows the current database path, watch targets, discovered memory targets, and recent Memrok activity +- `/memrok scan-memory` scans configured `MEMORY.md` files and `memory/` directories now +- `/memrok scan-memory force` reruns memory bootstrap even for files that were already bootstrapped +- `/memrok flush-sessions` runs transcript scribing immediately for any pending session chunks already seen by the watcher +- `/memrok index-sessions` replays unread session JSONL deltas from watched session paths +- `/memrok index-sessions full` rescans full watched session JSONL files from disk ## Privacy & Data Flow Memrok is local-first, but not magically offline. -- **Local database:** Memrok stores memory in a local SQLite database at `~/.memrok/memrok.db` by default. -- **Transcript and file access:** it watches OpenClaw session directories and any configured `watchPaths`. If bootstrap is enabled, it may also scan workspace Markdown files. +- **Local database:** Memrok stores memory in a local SQLite database under the active OpenClaw state directory, typically `~/.openclaw/plugins/memrok/memrok.db`. +- **Transcript and file access:** it watches OpenClaw session directories by default and any configured `watchPaths`. If bootstrap is enabled, it may also scan `MEMORY.md` and `memory/` across configured OpenClaw agents. - **Default posture:** bootstrap is disabled by default; broad file scanning should be an explicit choice. - **Remote model providers:** if you configure a remote provider for scribe passes, transcript and file content will be sent to that provider as part of normal operation. - **Risk controls:** narrow `watchPaths`, disable bootstrap if you do not want broad file scanning, prefer local models where available, and consider disabling the reflective scribe if you want to minimize exfiltration risk. -- **Operational hygiene:** treat `~/.memrok/memrok.db` as sensitive data; back it up and secure it accordingly. +- **Operational hygiene:** treat `~/.openclaw/plugins/memrok/memrok.db` as sensitive data; back it up and secure it accordingly. ## Hardened / low-exfiltration posture If you want a stricter setup: -- set `scribeProvider` / `scribeModel` to a local provider when possible +- keep OpenClaw’s default provider/model local when possible, or set `scribeProvider` / `scribeModel` explicitly for Memrok - keep `bootstrap.enabled` off unless you explicitly need seeding from Markdown files - narrow `watchPaths` to only what Memrok should ingest - disable or narrow reflection if you want less model-side synthesis @@ -146,29 +136,32 @@ packages/ ### Configuration -Most options are optional, but scribe provider/model should be set explicitly through OpenClaw config instead of relying on Memrok-owned defaults. +Most options are optional. Memrok inherits OpenClaw’s default provider/model unless you override them here. | Option | Type | Default | Description | | -------------------------- | -------- | -------------- | ---------------------------------------- | -| `dbPath` | string | state dir | Path to the SQLite database | -| `scribeProvider` | string | none | Model provider for the transcript scribe; set explicitly in OpenClaw config | -| `scribeModel` | string | none | Model for the transcript scribe; set explicitly in OpenClaw config | -| `watchPaths` | string[] | session dirs | Additional transcript paths to watch | +| `dbPath` | string | `${OPENCLAW_STATE_DIR}/plugins/memrok/memrok.db` | Path to the SQLite database | +| `scribeProvider` | string | OpenClaw default | Override provider for the transcript scribe | +| `scribeModel` | string | OpenClaw default | Override model for the transcript scribe | +| `watchPaths` | string[] | auto-detected OpenClaw session dirs | Additional transcript paths to watch | | `bootstrap.enabled` | boolean | false | Opt in to seeding from existing Markdown memory files | +| `bootstrap.scanConfiguredAgents` | boolean | true | Scan `MEMORY.md` and `memory/` across configured OpenClaw agents | +| `bootstrap.memoryDirs` | string[] | auto-discovered agent memory dirs | Extra memory directories to seed from | +| `bootstrap.memoryIndexes` | string[] | auto-discovered agent `MEMORY.md` files | Extra `MEMORY.md` files to seed from | | `deltaThreshold` | number | 20 | Messages before triggering consolidation | | `idleMinutes` | number | 15 | Quiet time required before scribe runs | | `tokenBudget` | number | 1000 | Max tokens for injected memory headers | | `reflection.enabled` | boolean | true | Enable the reflective scribe | | `reflection.deltaPasses` | number | 5 | Transcript passes between reflections | | `reflection.cooldownHours` | number | 24 | Minimum hours between reflection runs | -| `reflection.model` | string | scribeModel | Override model for reflection; otherwise inherits explicit transcript model | -| `reflection.provider` | string | scribeProvider | Override provider for reflection; otherwise inherits explicit transcript provider | +| `reflection.model` | string | OpenClaw / scribe model | Override model for reflection | +| `reflection.provider` | string | OpenClaw / scribe provider | Override provider for reflection | ## Status Deployed as an OpenClaw context engine plugin with dual-scribe architecture. 93 tests across the monorepo. -Memrok also writes a small health snapshot to `~/.memrok/memrok.status.json`, including recent transcript-scribe, reflective-scribe, and injection activity plus last error and node count. +Memrok also writes a small health snapshot under the OpenClaw state dir, typically `~/.openclaw/plugins/memrok/memrok.status.json`, including recent transcript-scribe, reflective-scribe, and injection activity plus last error and node count. For the full technical design, see [`docs/architecture.md`](docs/architecture.md). diff --git a/packages/openclaw-plugin/SKILL.md b/packages/openclaw-plugin/SKILL.md index ff47e58..ff64505 100644 --- a/packages/openclaw-plugin/SKILL.md +++ b/packages/openclaw-plugin/SKILL.md @@ -13,30 +13,46 @@ Persistent memory layer for OpenClaw AI agents. Watches conversation transcripts openclaw plugins install clawhub:memrok ``` +Memrok configuration lives under `plugins.entries.memrok.config`. After install, restart the OpenClaw gateway so the running process picks up the plugin. + ## Configuration -All options are optional — defaults work without tuning. +All options are optional. Memrok uses OpenClaw's configured provider/model by default and stores its local state under the active OpenClaw state directory, typically `~/.openclaw/plugins/memrok/`. | Option | Description | Default | |--------|-------------|---------| -| `scribeProvider` | LLM provider for knowledge extraction | `anthropic` | -| `scribeModel` | Model for scribe passes | `claude-sonnet-4-6` | +| `scribeProvider` | Override provider for knowledge extraction | OpenClaw default | +| `scribeModel` | Override model for scribe passes | OpenClaw default | | `watchPaths` | Directories to watch for transcript changes | auto-detected session dirs | +| `bootstrap.enabled` | Scan `MEMORY.md` and `memory/` at startup | `false` | +| `bootstrap.scanConfiguredAgents` | Include all configured OpenClaw agents in bootstrap scans | `true` | +| `bootstrap.memoryDirs` | Additional memory directories to scan | auto-discovered agent memory dirs | +| `bootstrap.memoryIndexes` | Additional `MEMORY.md` files to scan | auto-discovered agent `MEMORY.md` files | | `tokenBudget` | Max tokens for injected context header | `1000` | | `deltaThreshold` | Message count before triggering scribe | `20` | | `idleMinutes` | Quiet time required before scribe runs | `15` | +## Commands + +- `/memrok status` shows current Memrok paths, targets, and recent activity +- `/memrok scan-memory` scans configured Markdown memory sources now +- `/memrok scan-memory force` rescans already-bootstrapped Markdown memory sources +- `/memrok flush-sessions` runs transcript scribing immediately for pending watcher chunks +- `/memrok index-sessions` indexes unread watched session JSONL deltas from disk +- `/memrok index-sessions full` replays full watched session JSONL files from disk + ## What It Does 1. **Watches** OpenClaw session transcript files for changes -2. **Extracts** knowledge via scribe passes (entities, relationships, preferences, patterns) -3. **Stores** in a local SQLite knowledge graph -4. **Injects** relevant context as a header into every agent session turn +2. **Discovers** configured OpenClaw agent workspaces for bootstrap scans when enabled +3. **Extracts** knowledge via scribe passes (entities, relationships, preferences, patterns) +4. **Stores** in a local SQLite knowledge graph under the OpenClaw state dir +5. **Injects** relevant context as a header into every agent session turn ## Requirements - OpenClaw v0.30+ -- A configured LLM provider for scribe (uses the OpenClaw runtime's model plumbing) +- A configured LLM provider for scribe unless you override Memrok with explicit provider/model values ## Links diff --git a/packages/openclaw-plugin/openclaw.plugin.json b/packages/openclaw-plugin/openclaw.plugin.json index e875dab..9d8a2da 100644 --- a/packages/openclaw-plugin/openclaw.plugin.json +++ b/packages/openclaw-plugin/openclaw.plugin.json @@ -3,42 +3,147 @@ "kind": "context-engine", "name": "Memrok Memory Layer", "description": "Memory with judgment. A graph-based curation layer for OpenClaw, complementing built-in recall and dreaming with explicit supersession, expiry, and topic-aware selection.", + "uiHints": { + "dbPath": { + "label": "Database Path", + "help": "Path to the Memrok SQLite database. Defaults under the active OpenClaw state directory at plugins/memrok/memrok.db." + }, + "scribeProvider": { + "label": "Transcript Provider", + "help": "Optional provider override for transcript scribing. Leave empty to inherit OpenClaw's active default provider." + }, + "scribeModel": { + "label": "Transcript Model", + "help": "Optional model override for transcript scribing. Leave empty to inherit OpenClaw's active default model." + }, + "watchPaths": { + "label": "Watch Paths", + "help": "Additional transcript paths to watch. By default Memrok uses OpenClaw's configured session directories." + }, + "bootstrap.enabled": { + "label": "Enable Bootstrap", + "help": "When enabled, seed the graph from Markdown memory files before normal transcript indexing." + }, + "bootstrap.scanConfiguredAgents": { + "label": "Scan Configured Agents", + "help": "Include MEMORY.md and memory/ directories from discovered OpenClaw agent workspaces." + }, + "bootstrap.memoryDirs": { + "label": "Extra Memory Dirs", + "help": "Additional Markdown memory directories to include in bootstrap scans." + }, + "bootstrap.memoryIndexes": { + "label": "Extra Memory Indexes", + "help": "Additional MEMORY.md files to include in bootstrap scans." + }, + "reflection.provider": { + "label": "Reflection Provider", + "help": "Optional provider override for reflective scribing. Leave empty to inherit the transcript/OpenClaw default provider." + }, + "reflection.model": { + "label": "Reflection Model", + "help": "Optional model override for reflective scribing. Leave empty to inherit the transcript/OpenClaw default model." + } + }, "configSchema": { "type": "object", "additionalProperties": false, "properties": { - "dbPath": { "type": "string" }, - "scribeProvider": { "type": "string" }, - "scribeModel": { "type": "string" }, + "dbPath": { + "type": "string", + "description": "Path to the Memrok SQLite database. Defaults under OPENCLAW_STATE_DIR/plugins/memrok/memrok.db." + }, + "scribeProvider": { + "type": "string", + "description": "Optional provider override for transcript scribing. Leave unset to inherit OpenClaw's default provider." + }, + "scribeModel": { + "type": "string", + "description": "Optional model override for transcript scribing. Leave unset to inherit OpenClaw's default model." + }, "reflection": { "type": "object", "additionalProperties": false, "properties": { - "enabled": { "type": "boolean" }, - "deltaPasses": { "type": "number" }, - "cooldownHours": { "type": "number" }, - "provider": { "type": "string" }, - "model": { "type": "string" } + "enabled": { + "type": "boolean", + "description": "Enable reflective scribing over the curated graph." + }, + "deltaPasses": { + "type": "number", + "description": "Number of transcript scribe passes required before reflection is eligible." + }, + "cooldownHours": { + "type": "number", + "description": "Minimum hours between reflective scribe runs." + }, + "provider": { + "type": "string", + "description": "Optional provider override for reflection. Leave unset to inherit transcript/OpenClaw defaults." + }, + "model": { + "type": "string", + "description": "Optional model override for reflection. Leave unset to inherit transcript/OpenClaw defaults." + } } }, "bootstrap": { "type": "object", "additionalProperties": false, "properties": { - "enabled": { "type": "boolean" }, - "memoryDir": { "type": "string" }, - "memoryIndex": { "type": "string" }, - "maxAgeDays": { "type": "number" }, - "delayMs": { "type": "number" } + "enabled": { + "type": "boolean", + "description": "Enable bootstrap scans over Markdown memory sources." + }, + "memoryDir": { + "type": "string", + "description": "Legacy single memory directory override." + }, + "memoryDirs": { + "type": "array", + "items": { "type": "string" }, + "description": "Additional memory directories to include in bootstrap scans." + }, + "memoryIndex": { + "type": "string", + "description": "Legacy single MEMORY.md override." + }, + "memoryIndexes": { + "type": "array", + "items": { "type": "string" }, + "description": "Additional MEMORY.md files to include in bootstrap scans." + }, + "scanConfiguredAgents": { + "type": "boolean", + "description": "Discover configured OpenClaw agent workspaces and scan their MEMORY.md and memory/ sources." + }, + "maxAgeDays": { + "type": "number", + "description": "Skip Markdown memory files older than this age unless forced." + }, + "delayMs": { + "type": "number", + "description": "Delay between bootstrap file scribe calls." + } } }, "watchPaths": { "type": "array", - "items": { "type": "string" } + "items": { "type": "string" }, + "description": "Additional transcript paths to watch. By default Memrok uses OpenClaw session directories." + }, + "deltaThreshold": { + "type": "number", + "description": "Message threshold before consolidation is eligible." + }, + "idleMinutes": { + "type": "number", + "description": "Idle time required before transcript scribing runs." }, - "deltaThreshold": { "type": "number" }, - "idleMinutes": { "type": "number" }, - "tokenBudget": { "type": "number" } + "tokenBudget": { + "type": "number", + "description": "Maximum token budget for Memrok's injected context header." + } } } } diff --git a/packages/openclaw-plugin/package.json b/packages/openclaw-plugin/package.json index 3534260..85cfbf7 100644 --- a/packages/openclaw-plugin/package.json +++ b/packages/openclaw-plugin/package.json @@ -36,7 +36,6 @@ "test": "vitest run" }, "dependencies": { - "better-sqlite3": "^11.9.1", "chokidar": "^4.0.3" }, "devDependencies": { diff --git a/packages/openclaw-plugin/src/index.ts b/packages/openclaw-plugin/src/index.ts index a5de7c8..1721d3d 100644 --- a/packages/openclaw-plugin/src/index.ts +++ b/packages/openclaw-plugin/src/index.ts @@ -11,7 +11,10 @@ export type { ContextEngine, ContextHeader, MemrokPluginConfig, - PluginApi, - PluginRegistration, - ResolvedConfig, -} from './types.js'; + PluginApi, + PluginCommandContext, + PluginCommandDefinition, + PluginCommandResult, + PluginRegistration, + ResolvedConfig, + } from './types.js'; diff --git a/packages/openclaw-plugin/src/plugin.test.ts b/packages/openclaw-plugin/src/plugin.test.ts index a069e09..7351cd5 100644 --- a/packages/openclaw-plugin/src/plugin.test.ts +++ b/packages/openclaw-plugin/src/plugin.test.ts @@ -1,20 +1,22 @@ import { afterEach, describe, it, vi } from 'vitest'; import assert from 'node:assert/strict'; -import Database from 'better-sqlite3'; -import { mkdtempSync, rmSync } from 'node:fs'; +import { DatabaseSync } from 'node:sqlite'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { createStore } from '@memrok/store'; -import { createPluginRegistration, resolveConfig, runReflectionPass, shouldRunReflection } from './plugin.js'; -import type { ContextEngine, PluginApi, PluginService } from './types.js'; +import { createModelCaller, createPluginRegistration, resolveConfig, runReflectionPass, shouldRunReflection } from './plugin.js'; +import type { ContextEngine, PluginApi, PluginCommandDefinition, PluginService } from './types.js'; function createApi(overrides: Partial = {}): { api: PluginApi; services: PluginService[]; factories: Map ContextEngine>; + commands: PluginCommandDefinition[]; } { const services: PluginService[] = []; const factories = new Map ContextEngine>(); + const commands: PluginCommandDefinition[] = []; const api: PluginApi = { pluginConfig: {}, logger: { @@ -29,9 +31,12 @@ function createApi(overrides: Partial = {}): { registerService(service) { services.push(service); }, + registerCommand(command) { + commands.push(command); + }, ...overrides, }; - return { api, services, factories }; + return { api, services, factories, commands }; } afterEach(() => { @@ -48,7 +53,7 @@ describe('openclaw plugin orchestration', () => { assert.equal(resolved.idleMinutes, 15); assert.equal(resolved.tokenBudget, 1000); assert.equal(resolved.bootstrap.enabled, false); - assert.ok(resolved.dbPath.endsWith('/.memrok/memrok.db')); + assert.ok(resolved.dbPath.endsWith('/.openclaw/plugins/memrok/memrok.db')); }); it('resolves reflection config defaults', () => { @@ -169,6 +174,297 @@ describe('openclaw plugin orchestration', () => { rmSync(dir, { recursive: true, force: true }); } }); + + it('discovers memory targets for multiple configured OpenClaw agents', () => { + const stateDir = mkdtempSync(join(tmpdir(), 'memrok-openclaw-state-')); + const oldStateDir = process.env.OPENCLAW_STATE_DIR; + process.env.OPENCLAW_STATE_DIR = stateDir; + + try { + mkdirSync(join(stateDir, 'agents', 'alpha', 'memory'), { recursive: true }); + mkdirSync(join(stateDir, 'agents', 'beta', 'memory'), { recursive: true }); + + const { api } = createApi({ + pluginConfig: { + dbPath: join(stateDir, 'memrok.db'), + bootstrap: { + enabled: true, + }, + }, + }); + + const runtime = createPluginRegistration(api); + const targets = runtime.describeBootstrapTargets(); + + assert.deepEqual(targets.memoryDirs, [ + join(stateDir, 'agents', 'alpha', 'memory'), + join(stateDir, 'agents', 'beta', 'memory'), + ]); + assert.deepEqual(targets.memoryIndexes, [ + join(stateDir, 'agents', 'alpha', 'MEMORY.md'), + join(stateDir, 'agents', 'beta', 'MEMORY.md'), + ]); + runtime.store.close(); + } finally { + if (oldStateDir === undefined) delete process.env.OPENCLAW_STATE_DIR; + else process.env.OPENCLAW_STATE_DIR = oldStateDir; + rmSync(stateDir, { recursive: true, force: true }); + } + }); + + it('registers a memrok command surface for manual operations', async () => { + const dir = mkdtempSync(join(tmpdir(), 'memrok-command-')); + try { + const { api, commands } = createApi({ + pluginConfig: { + dbPath: join(dir, 'memrok.db'), + watchPaths: [dir], + }, + }); + + const runtime = createPluginRegistration(api); + assert.equal(commands.length, 1); + assert.equal(commands[0]?.name, 'memrok'); + + const help = await commands[0]!.handler({ args: 'help' }); + assert.match(help.text, /scan-memory/); + + const status = await commands[0]!.handler({ args: 'status' }); + assert.match(status.text, /Memrok/); + runtime.store.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('omits explicit provider and model so OpenClaw defaults can apply', async () => { + const dir = mkdtempSync(join(tmpdir(), 'memrok-model-defaults-')); + const runEmbeddedPiAgent = vi.fn(async () => ({ + payloads: [{ text: '{"pass_id":"scribe-defaults","mutations":[]}' }], + })); + + try { + const { api } = createApi({ + runtime: { + agent: { + runEmbeddedPiAgent, + resolveAgentWorkspaceDir: () => dir, + }, + }, + }); + + const caller = createModelCaller(api, resolveConfig({}, api)); + await caller('system', 'user'); + + const params = runEmbeddedPiAgent.mock.calls[0]?.[0] as Record; + assert.ok(params); + assert.equal('provider' in params, false); + assert.equal('model' in params, false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('bootstraps memory files on service start even when transcript passes already exist', async () => { + const dir = mkdtempSync(join(tmpdir(), 'memrok-bootstrap-start-')); + const dbPath = join(dir, 'memrok.db'); + const seededStore = createStore(dbPath); + + try { + mkdirSync(join(dir, 'memory'), { recursive: true }); + writeFileSync(join(dir, 'memory', 'note.md'), '# Existing memory\n'); + seededStore.applyPass({ + pass_id: 'transcript-existing', + source: 'session-1', + mutations: [ + { + operation: 'add', + layer: 'user', + category: 'fact', + key: 'user.fact.seeded', + value: 'already has transcript data', + }, + ], + }); + seededStore.close(); + + const runEmbeddedPiAgent = vi.fn(async () => ({ + payloads: [{ text: '{"pass_id":"bootstrap-pass","mutations":[]}' }], + })); + + const { api, services } = createApi({ + pluginConfig: { + dbPath, + watchPaths: [dir], + bootstrap: { + enabled: true, + delayMs: 0, + scanConfiguredAgents: false, + }, + }, + runtime: { + agent: { + runEmbeddedPiAgent, + resolveAgentWorkspaceDir: () => dir, + }, + }, + }); + + createPluginRegistration(api); + await services[0]!.start({ stateDir: dir }); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.ok(runEmbeddedPiAgent.mock.calls.length >= 1); + await services[0]!.stop(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('bounds full session reindex runs and reports remaining files', async () => { + const dir = mkdtempSync(join(tmpdir(), 'memrok-index-limit-')); + const sessionsDir = join(dir, 'sessions'); + mkdirSync(sessionsDir, { recursive: true }); + const oldStateDir = process.env.OPENCLAW_STATE_DIR; + process.env.OPENCLAW_STATE_DIR = dir; + + try { + for (let index = 0; index < 30; index++) { + writeFileSync( + join(sessionsDir, `session-${index.toString().padStart(2, '0')}.jsonl`), + '{"type":"user","text":"hello"}\n', + ); + } + + let passCounter = 0; + const runEmbeddedPiAgent = vi.fn(async () => ({ + payloads: [{ text: `{"pass_id":"session-index-pass-${++passCounter}","mutations":[]}` }], + })); + + const { api } = createApi({ + pluginConfig: { + dbPath: join(dir, 'memrok.db'), + }, + runtime: { + agent: { + sessionDirs: [sessionsDir], + runEmbeddedPiAgent, + resolveAgentWorkspaceDir: () => dir, + }, + }, + }); + + const runtime = createPluginRegistration(api); + const result = await runtime.indexSessionFiles({ full: true }); + + assert.equal(result.filesConsidered, 30); + assert.equal(result.unreadCandidates, 30); + assert.equal(result.limitApplied, 25); + assert.equal(result.processed, 25); + assert.equal(result.remaining, 5); + runtime.store.close(); + } finally { + if (oldStateDir === undefined) delete process.env.OPENCLAW_STATE_DIR; + else process.env.OPENCLAW_STATE_DIR = oldStateDir; + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('advances the cursor after a full replay so unread replay does not immediately duplicate it', async () => { + const dir = mkdtempSync(join(tmpdir(), 'memrok-index-cursor-')); + const sessionsDir = join(dir, 'sessions'); + mkdirSync(sessionsDir, { recursive: true }); + writeFileSync(join(sessionsDir, 'session.jsonl'), '{"type":"user","text":"hello"}\n'); + const oldStateDir = process.env.OPENCLAW_STATE_DIR; + process.env.OPENCLAW_STATE_DIR = dir; + + let passCounter = 0; + const runEmbeddedPiAgent = vi.fn(async () => ({ + payloads: [{ text: `{"pass_id":"session-replay-pass-${++passCounter}","mutations":[]}` }], + })); + + try { + const { api } = createApi({ + pluginConfig: { + dbPath: join(dir, 'memrok.db'), + }, + runtime: { + agent: { + sessionDirs: [sessionsDir], + runEmbeddedPiAgent, + resolveAgentWorkspaceDir: () => dir, + }, + }, + }); + + const runtime = createPluginRegistration(api); + const fullReplay = await runtime.indexSessionFiles({ full: true, limit: 10 }); + const unreadReplay = await runtime.indexSessionFiles({ limit: 10 }); + + assert.equal(fullReplay.processed, 1); + assert.equal(fullReplay.unreadCandidates, 1); + assert.equal(unreadReplay.processed, 0); + assert.equal(unreadReplay.unreadCandidates, 0); + assert.equal(unreadReplay.skipped, 0); + runtime.store.close(); + } finally { + if (oldStateDir === undefined) delete process.env.OPENCLAW_STATE_DIR; + else process.env.OPENCLAW_STATE_DIR = oldStateDir; + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('prefilters unread session indexing to files with unread bytes before applying the limit', async () => { + const dir = mkdtempSync(join(tmpdir(), 'memrok-index-unread-filter-')); + const sessionsDir = join(dir, 'sessions'); + mkdirSync(sessionsDir, { recursive: true }); + const oldStateDir = process.env.OPENCLAW_STATE_DIR; + process.env.OPENCLAW_STATE_DIR = dir; + + const readPath = join(sessionsDir, 'older-read.jsonl'); + const unreadPath = join(sessionsDir, 'newer-unread.jsonl'); + writeFileSync(readPath, '{"type":"user","text":"read"}\n'); + writeFileSync(unreadPath, '{"type":"user","text":"unread"}\n'); + + let passCounter = 0; + const runEmbeddedPiAgent = vi.fn(async () => ({ + payloads: [{ text: `{"pass_id":"session-unread-pass-${++passCounter}","mutations":[]}` }], + })); + + try { + const { api } = createApi({ + pluginConfig: { + dbPath: join(dir, 'memrok.db'), + }, + runtime: { + agent: { + sessionDirs: [sessionsDir], + runEmbeddedPiAgent, + resolveAgentWorkspaceDir: () => dir, + }, + }, + }); + + const runtime = createPluginRegistration(api); + runtime.watcher.setCursor(readPath, readFileSync(readPath, 'utf-8').length); + runtime.watcher.saveCursors(); + + const result = await runtime.indexSessionFiles({ limit: 1 }); + + assert.equal(result.filesConsidered, 2); + assert.equal(result.unreadCandidates, 1); + assert.equal(result.processed, 1); + assert.equal(result.remaining, 0); + assert.equal(runEmbeddedPiAgent.mock.calls.length, 1); + const call = runEmbeddedPiAgent.mock.calls[0]?.[0] as { sessionFile?: string; prompt?: string }; + assert.ok(call); + assert.match(call.prompt ?? '', /unread/); + runtime.store.close(); + } finally { + if (oldStateDir === undefined) delete process.env.OPENCLAW_STATE_DIR; + else process.env.OPENCLAW_STATE_DIR = oldStateDir; + rmSync(dir, { recursive: true, force: true }); + } + }); }); describe('shouldRunReflection', () => { @@ -404,7 +700,7 @@ describe('runReflectionPass', () => { }); seededStore.close(); - const db = new Database(dbPath); + const db = new DatabaseSync(dbPath); db.prepare('UPDATE passes SET timestamp = ? WHERE pass_id = ?').run('2026-01-01 00:00:00', 'reflection-old'); db.prepare('UPDATE passes SET timestamp = ? WHERE pass_id = ?').run('2026-01-01 01:00:00', 'transcript-after-reflection'); db.close(); diff --git a/packages/openclaw-plugin/src/plugin.ts b/packages/openclaw-plugin/src/plugin.ts index 0600f8b..6dc6534 100644 --- a/packages/openclaw-plugin/src/plugin.ts +++ b/packages/openclaw-plugin/src/plugin.ts @@ -19,7 +19,6 @@ import type { ResolvedReflectionConfig, } from './types.js'; -const DEFAULT_DB_PATH = '~/.memrok/memrok.db'; const DEFAULT_DELTA_THRESHOLD = 20; const DEFAULT_IDLE_MINUTES = 15; const DEFAULT_TOKEN_BUDGET = 1000; @@ -30,6 +29,9 @@ const DEFAULT_REFLECTION_CHECK_INTERVAL_MS = 60_000; const DEFAULT_BOOTSTRAP_ENABLED = false; const DEFAULT_BOOTSTRAP_MAX_AGE_DAYS = 90; const DEFAULT_BOOTSTRAP_DELAY_MS = 10_000; +const DEFAULT_BOOTSTRAP_SCAN_CONFIGURED_AGENTS = true; +const DEFAULT_INDEX_SESSION_LIMIT = 100; +const DEFAULT_FULL_REINDEX_LIMIT = 25; function expandHome(input: string): string { if (input === '~') return homedir(); @@ -37,6 +39,64 @@ function expandHome(input: string): string { return resolve(input); } +function resolveOpenclawStateDir(env: NodeJS.ProcessEnv = process.env): string { + const explicit = env.OPENCLAW_STATE_DIR?.trim(); + return explicit ? expandHome(explicit) : join(homedir(), '.openclaw'); +} + +function resolveMemrokDataDir(api?: PluginApi): string { + const runtimeAgent = toRecord(api?.runtime?.agent); + const explicitStateDir = typeof runtimeAgent?.stateDir === 'string' ? runtimeAgent.stateDir : undefined; + const stateDir = explicitStateDir?.trim() + ? expandHome(explicitStateDir) + : resolveOpenclawStateDir(); + return join(stateDir, 'plugins', 'memrok'); +} + +function resolveMemrokTmpDir(api?: PluginApi): string { + return join(resolveMemrokDataDir(api), 'tmp'); +} + +function toRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? value as Record + : undefined; +} + +function toStringArray(value: unknown): string[] { + if (Array.isArray(value)) { + return value + .map((entry) => typeof entry === 'string' ? entry.trim() : '') + .filter(Boolean); + } + if (typeof value === 'string') { + return value + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean); + } + return []; +} + +function resolvePluginConfig( + raw: MemrokPluginConfig | Record | undefined, + api?: PluginApi, +): MemrokPluginConfig { + if (raw && Object.keys(raw).length > 0) { + return raw as MemrokPluginConfig; + } + + const rootConfig = toRecord(api?.config); + const plugins = toRecord(rootConfig?.plugins); + const entries = toRecord(plugins?.entries); + const pluginEntry = toRecord(entries?.memrok); + return (toRecord(pluginEntry?.config) ?? {}) as MemrokPluginConfig; +} + +function normalizeFileKey(filePath: string): string { + return resolve(filePath); +} + function extractText(content: Message['content']): string { if (typeof content === 'string') return content; if (Array.isArray(content)) { @@ -65,6 +125,11 @@ function recentContextFromMessages(messages: Message[]): string { } function defaultWatchPaths(api: PluginApi): string[] { + const discoveredSessionDirs = discoverSessionDirs(api); + if (discoveredSessionDirs.length > 0) { + return discoveredSessionDirs; + } + const agent = api.runtime?.agent; if (Array.isArray(agent?.sessionDirs) && agent.sessionDirs.length > 0) { return agent.sessionDirs.map(expandHome); @@ -72,12 +137,112 @@ function defaultWatchPaths(api: PluginApi): string[] { if (agent?.sessionsDir) { return [expandHome(agent.sessionsDir)]; } - return [join(homedir(), '.openclaw', 'agents')]; + return [join(resolveOpenclawStateDir(), 'agents')]; +} + +function discoverSessionDirs(api: PluginApi): string[] { + const sessionDirs = new Set(); + const addSessionDir = (candidate: string | undefined) => { + const trimmed = candidate?.trim(); + if (!trimmed) return; + sessionDirs.add(expandHome(trimmed)); + }; + + const runtimeAgent = api.runtime?.agent; + for (const sessionDir of runtimeAgent?.sessionDirs ?? []) { + addSessionDir(sessionDir); + } + addSessionDir(runtimeAgent?.sessionsDir); + + const agentsDir = join(resolveOpenclawStateDir(), 'agents'); + try { + const entries = readdirSync(agentsDir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const candidate = join(agentsDir, entry.name, 'sessions'); + if (existsSync(candidate)) { + sessionDirs.add(candidate); + } + } + } catch { + // Fall back to runtime-provided session directories if the state dir layout + // is unavailable in this environment. + } + + return [...sessionDirs].sort(); +} + +function resolveSessionReplayPaths(api: PluginApi, configuredWatchPaths: string[]): string[] { + const discoveredSessionDirs = discoverSessionDirs(api); + if (discoveredSessionDirs.length > 0) { + return discoveredSessionDirs; + } + + const broadFallback = join(resolveOpenclawStateDir(), 'agents'); + return configuredWatchPaths + .map(expandHome) + .filter((watchPath) => watchPath !== broadFallback); +} + +function discoverAgentRoots(api: PluginApi, workspaceDir: string): string[] { + const roots = new Set(); + const fallbacks = new Set(); + + const addRoot = (candidate: string | undefined) => { + const trimmed = candidate?.trim(); + if (!trimmed) return; + roots.add(expandHome(trimmed)); + }; + + const addFallbackRoot = (candidate: string | undefined) => { + const trimmed = candidate?.trim(); + if (!trimmed) return; + fallbacks.add(expandHome(trimmed)); + }; + + const addSessionPath = (candidate: string | undefined) => { + const trimmed = candidate?.trim(); + if (!trimmed) return; + const resolved = expandHome(trimmed); + if (basename(resolved) === 'sessions') { + roots.add(dirname(resolved)); + return; + } + roots.add(resolved); + }; + + const runtimeAgent = api.runtime?.agent; + for (const sessionDir of runtimeAgent?.sessionDirs ?? []) { + addSessionPath(sessionDir); + } + addSessionPath(runtimeAgent?.sessionsDir); + addFallbackRoot(runtimeAgent?.resolveAgentWorkspaceDir?.(api.config)); + addFallbackRoot(workspaceDir); + + const agentsDir = join(resolveOpenclawStateDir(), 'agents'); + try { + const entries = readdirSync(agentsDir, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isDirectory()) { + roots.add(join(agentsDir, entry.name)); + } + } + } catch { + // No global agent registry available — fall back to runtime-discovered roots. + } + + if (roots.size === 0) { + for (const fallback of fallbacks) { + roots.add(fallback); + } + } + + return [...roots].sort(); } -/** Remove stale memrok-scribe-*.jsonl files from ~/.memrok/tmp/ on startup. */ -function cleanupStaleTmpFiles(): void { - const tmpDir = join(homedir(), '.memrok', 'tmp'); +/** Remove stale memrok-scribe-*.jsonl files from the plugin temp directory on startup. */ +function cleanupStaleTmpFiles(api?: PluginApi): void { + const tmpDir = resolveMemrokTmpDir(api); try { const entries = readdirSync(tmpDir); let cleaned = 0; @@ -101,7 +266,7 @@ export function resolveConfig( raw: MemrokPluginConfig | Record | undefined, api?: PluginApi, ): ResolvedConfig { - const config = (raw ?? {}) as MemrokPluginConfig; + const config = resolvePluginConfig(raw, api); const scribeProvider = config.scribeProvider; const scribeModel = config.scribeModel; const reflection: ResolvedReflectionConfig = { @@ -114,12 +279,21 @@ export function resolveConfig( const bootstrap: ResolvedBootstrapConfig = { enabled: config.bootstrap?.enabled ?? DEFAULT_BOOTSTRAP_ENABLED, memoryDir: config.bootstrap?.memoryDir, + memoryDirs: [ + ...toStringArray(config.bootstrap?.memoryDirs), + ...toStringArray(config.bootstrap?.memoryDir), + ], memoryIndex: config.bootstrap?.memoryIndex, + memoryIndexes: [ + ...toStringArray(config.bootstrap?.memoryIndexes), + ...toStringArray(config.bootstrap?.memoryIndex), + ], + scanConfiguredAgents: config.bootstrap?.scanConfiguredAgents ?? DEFAULT_BOOTSTRAP_SCAN_CONFIGURED_AGENTS, maxAgeDays: config.bootstrap?.maxAgeDays ?? DEFAULT_BOOTSTRAP_MAX_AGE_DAYS, delayMs: config.bootstrap?.delayMs ?? DEFAULT_BOOTSTRAP_DELAY_MS, }; return { - dbPath: expandHome(config.dbPath ?? DEFAULT_DB_PATH), + dbPath: expandHome(config.dbPath ?? join(resolveMemrokDataDir(api), 'memrok.db')), scribeProvider, scribeModel, watchPaths: (config.watchPaths?.length ? config.watchPaths : defaultWatchPaths(api as PluginApi)).map(expandHome), @@ -175,7 +349,7 @@ export function createModelCaller(api: PluginApi, config: ResolvedConfig, inputL throw new Error('api.runtime.agent.runEmbeddedPiAgent not available'); } - const tmpDir = join(homedir(), '.memrok', 'tmp'); + const tmpDir = resolveMemrokTmpDir(api); mkdirSync(tmpDir, { recursive: true }); const sessionId = `memrok-scribe-${Date.now()}`; const sessionFile = join(tmpDir, `${sessionId}.jsonl`); @@ -195,8 +369,8 @@ export function createModelCaller(api: PluginApi, config: ResolvedConfig, inputL prompt: fullPrompt, timeoutMs: 60_000, runId: `memrok-scribe-${Date.now()}`, - provider: config.scribeProvider, - model: config.scribeModel, + ...(config.scribeProvider ? { provider: config.scribeProvider } : {}), + ...(config.scribeModel ? { model: config.scribeModel } : {}), disableTools: true, }); } finally { @@ -247,34 +421,189 @@ function scanMarkdownFiles(dir: string): string[] { return results.sort(); } +function scanJsonlFiles(dir: string): string[] { + const results: string[] = []; + function walk(d: string): void { + let entries; + try { + entries = readdirSync(d, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const fullPath = join(d, entry.name); + if (entry.isDirectory()) { + walk(fullPath); + } else if (entry.isFile() && extname(entry.name) === '.jsonl') { + results.push(fullPath); + } + } + } + walk(dir); + return results.sort(); +} + +function resolveBootstrapTargets( + bootstrapConfig: ResolvedBootstrapConfig, + api: PluginApi, + workspaceDir: string, +): { memoryDirs: string[]; memoryIndexes: string[] } { + const memoryDirs = new Set(); + const memoryIndexes = new Set(); + + for (const candidate of bootstrapConfig.memoryDirs) { + memoryDirs.add(expandHome(candidate)); + } + for (const candidate of bootstrapConfig.memoryIndexes) { + memoryIndexes.add(expandHome(candidate)); + } + + if (bootstrapConfig.scanConfiguredAgents) { + for (const agentRoot of discoverAgentRoots(api, workspaceDir)) { + memoryDirs.add(join(agentRoot, 'memory')); + memoryIndexes.add(join(agentRoot, 'MEMORY.md')); + } + } + + if (memoryDirs.size === 0 && memoryIndexes.size === 0) { + memoryDirs.add(bootstrapConfig.memoryDir + ? expandHome(bootstrapConfig.memoryDir) + : join(workspaceDir, 'memory')); + memoryIndexes.add(bootstrapConfig.memoryIndex + ? expandHome(bootstrapConfig.memoryIndex) + : join(workspaceDir, 'MEMORY.md')); + } + + return { + memoryDirs: [...memoryDirs].sort(), + memoryIndexes: [...memoryIndexes].sort(), + }; +} + +function formatTimestamp(value: string | null | undefined): string { + return value ? new Date(value).toISOString() : 'never'; +} + +type MemrokCommand = + | { kind: 'status' } + | { kind: 'scan-memory'; force: boolean } + | { kind: 'flush-sessions' } + | { kind: 'index-sessions'; full: boolean; limit?: number } + | { kind: 'help'; error?: string }; + +function parseCommandArgs(rawArgs: string | undefined): MemrokCommand { + const tokens = (rawArgs ?? '') + .trim() + .split(/\s+/) + .map((token) => token.trim()) + .filter(Boolean); + + if (tokens.length === 0 || tokens[0] === 'status') { + return tokens.length <= 1 + ? { kind: 'status' } + : { kind: 'help', error: '`/memrok status` does not accept extra arguments.' }; + } + + if (tokens[0] === 'scan-memory' || tokens[0] === 'bootstrap') { + const force = tokens.slice(1).includes('force'); + const unexpected = tokens.slice(1).filter((token) => token !== 'force'); + return unexpected.length === 0 + ? { kind: 'scan-memory', force } + : { kind: 'help', error: '`/memrok scan-memory` accepts only the optional `force` flag.' }; + } + + if (tokens[0] === 'flush-sessions' || tokens[0] === 'trigger') { + return tokens.length === 1 + ? { kind: 'flush-sessions' } + : { kind: 'help', error: '`/memrok flush-sessions` does not accept extra arguments.' }; + } + + if (tokens[0] === 'index-sessions' || tokens[0] === 'reindex-sessions') { + let full = false; + let limit: number | undefined; + const unexpected: string[] = []; + + for (let i = 1; i < tokens.length; i++) { + const token = tokens[i]!; + if (token === 'full') { + full = true; + continue; + } + if (token === 'limit' && i + 1 < tokens.length) { + const value = Number.parseInt(tokens[i + 1]!, 10); + if (Number.isFinite(value) && value > 0) { + limit = value; + i++; + continue; + } + } + if (token.startsWith('limit=')) { + const value = Number.parseInt(token.slice('limit='.length), 10); + if (Number.isFinite(value) && value > 0) { + limit = value; + continue; + } + } + unexpected.push(token); + } + + return unexpected.length === 0 + ? { kind: 'index-sessions', full, limit } + : { kind: 'help', error: '`/memrok index-sessions` accepts optional `full` and `limit=` flags.' }; + } + + if (tokens[0] === 'help') { + return { kind: 'help' }; + } + + return { + kind: 'help', + error: `Unknown subcommand \`${tokens[0]}\`. Supported: status, scan-memory, flush-sessions, index-sessions, help.`, + }; +} + +function buildHelpText(error?: string): string { + const lines = [ + '**Memrok**', + 'Commands:', + '- `/memrok status` show recent Memrok activity and discovered targets', + '- `/memrok scan-memory [force]` scan configured `MEMORY.md` and `memory/` files now', + '- `/memrok flush-sessions` run transcript scribing immediately for pending session changes', + '- `/memrok index-sessions [full] [limit=]` index session JSONL files now; `full` replays complete files', + '- `/memrok help` show this help', + ]; + return error ? [`Error: ${error}`, '', ...lines].join('\n') : lines.join('\n'); +} + export async function runBootstrap( store: ArchiveStore & ArtifactStore & GraphStore, scribe: ScribeInterface, bootstrapConfig: ResolvedBootstrapConfig, + api: PluginApi, workspaceDir: string, -): Promise { - const memoryDir = bootstrapConfig.memoryDir - ? expandHome(bootstrapConfig.memoryDir) - : join(workspaceDir, 'memory'); - const memoryIndex = bootstrapConfig.memoryIndex - ? expandHome(bootstrapConfig.memoryIndex) - : join(workspaceDir, 'MEMORY.md'); - - // Collect candidate files + options?: { force?: boolean }, +): Promise<{ filesConsidered: number; processed: number; skipped: number; failed: number }> { + const targets = resolveBootstrapTargets(bootstrapConfig, api, workspaceDir); + const force = options?.force ?? false; + const filePaths: string[] = []; - if (existsSync(memoryDir)) { - filePaths.push(...scanMarkdownFiles(memoryDir)); + for (const memoryDir of targets.memoryDirs) { + if (existsSync(memoryDir)) { + filePaths.push(...scanMarkdownFiles(memoryDir)); + } } - if (existsSync(memoryIndex) && !filePaths.includes(memoryIndex)) { - filePaths.push(memoryIndex); + for (const memoryIndex of targets.memoryIndexes) { + if (existsSync(memoryIndex) && !filePaths.includes(memoryIndex)) { + filePaths.push(memoryIndex); + } } if (filePaths.length === 0) { console.log('[memrok:bootstrap] No memory files found, skipping'); - return; + return { filesConsidered: 0, processed: 0, skipped: 0, failed: 0 }; } - // Build set of already-bootstrapped filenames from pass sources + // Build set of already-bootstrapped filenames from pass sources. const existingPasses = store.listPasses(); const bootstrappedFiles = new Set( existingPasses @@ -292,30 +621,27 @@ export async function runBootstrap( let failed = 0; for (const filePath of filePaths) { - const fileKey = relative(workspaceDir, filePath); + const fileKey = normalizeFileKey(filePath); + const legacyFileKey = relative(workspaceDir, filePath); - // Skip already bootstrapped - if (bootstrappedFiles.has(fileKey)) { + if (!force && (bootstrappedFiles.has(fileKey) || bootstrappedFiles.has(legacyFileKey))) { console.log(`[memrok:bootstrap] Skipping ${fileKey} (already bootstrapped)`); skipped++; continue; } - // Skip files older than maxAgeDays try { const stat = statSync(filePath); - if (now - stat.mtimeMs > maxAgeMs) { + if (!force && now - stat.mtimeMs > maxAgeMs) { console.log(`[memrok:bootstrap] Skipping ${fileKey} (older than ${bootstrapConfig.maxAgeDays} days)`); skipped++; continue; } } catch { - // If we can't stat, skip skipped++; continue; } - // Read content let content: string; try { content = readFileSync(filePath, 'utf-8'); @@ -330,7 +656,6 @@ export async function runBootstrap( continue; } - // Run through scribe try { const pass = await scribe.callModel(content); const result = persistObservationDrivenPass({ @@ -359,13 +684,13 @@ export async function runBootstrap( continue; } - // Rate limit between files (skip delay after last file) if (bootstrapConfig.delayMs > 0 && filePath !== filePaths[filePaths.length - 1]) { await new Promise((resolve) => setTimeout(resolve, bootstrapConfig.delayMs)); } } console.log(`[memrok:bootstrap] Done: ${processed} processed, ${skipped} skipped, ${failed} failed`); + return { filesConsidered: filePaths.length, processed, skipped, failed }; } export interface PluginRuntimeState { @@ -373,6 +698,20 @@ export interface PluginRuntimeState { injector: Injector; watcher: TranscriptWatcher; consolidation: ConsolidationEngine; + status: StatusTracker; + config: ResolvedConfig; + scanMemory(force?: boolean): Promise<{ filesConsidered: number; processed: number; skipped: number; failed: number }>; + flushPendingSessions(): Promise; + indexSessionFiles(options?: { full?: boolean; limit?: number }): Promise<{ + filesConsidered: number; + unreadCandidates: number; + processed: number; + skipped: number; + failed: number; + remaining: number; + limitApplied: number; + }>; + describeBootstrapTargets(): { memoryDirs: string[]; memoryIndexes: string[] }; } export function createContextEngine(runtime: PluginRuntimeState): ContextEngine { @@ -467,6 +806,7 @@ export async function runReflectionPass(params: { export function createPluginRegistration(api: PluginApi): PluginRuntimeState { const config = resolveConfig(api.pluginConfig, api); + cleanupStaleTmpFiles(api); mkdirSync(dirname(config.dbPath), { recursive: true }); const store = createStore(config.dbPath); const status = new StatusTracker(config.dbPath); @@ -495,14 +835,27 @@ export function createPluginRegistration(api: PluginApi): PluginRuntimeState { { systemPrompt: REFLECTION_SYSTEM_PROMPT }, ); - const watcher = new TranscriptWatcher({ paths: config.watchPaths }); + const watcher = new TranscriptWatcher( + { paths: config.watchPaths }, + join(dirname(config.dbPath), '.memrok-cursors.json'), + ); const consolidation = new ConsolidationEngine({ deltaThreshold: config.deltaThreshold, idleMinutes: config.idleMinutes, }); - const pendingTranscriptChunks: string[] = []; - let lastSourceProcessed: string | null = null; + const pendingTranscriptChunks: Array<{ source: string; content: string }> = []; let reflectionCheckTimer: ReturnType | null = null; + let sessionIndexRunInFlight: Promise<{ + filesConsidered: number; + unreadCandidates: number; + processed: number; + skipped: number; + failed: number; + remaining: number; + limitApplied: number; + }> | null = null; + const resolveWorkspaceDir = () => + api.runtime?.agent?.resolveAgentWorkspaceDir?.(api.config) ?? process.cwd(); // Reflection state — recover from DB so restarts don't reset the counters const allPasses = store.listPasses(); @@ -518,6 +871,10 @@ export function createPluginRegistration(api: PluginApi): PluginRuntimeState { if (reflectionRunInFlight) { return reflectionRunInFlight; } + // TODO: Investigate why the periodic reflection timer path does not reliably + // trigger in the fake-timer test case (`runs reflection from the timer after + // cooldown elapses without a new transcript pass`). Install/load behavior is + // correct, but the timer-driven test still fails intermittently. if (!shouldRunReflection(passesSinceReflection, lastReflectionTime, config.reflection)) return; reflectionRunInFlight = (async () => { try { @@ -533,18 +890,20 @@ export function createPluginRegistration(api: PluginApi): PluginRuntimeState { return reflectionRunInFlight; }; - const runScribePass = async () => { - if (pendingTranscriptChunks.length === 0) return; - const transcript = pendingTranscriptChunks.join('\n'); + const runTranscriptScribeForSource = async ( + source: string, + transcript: string, + metadata?: Record, + ): Promise => { try { const pass = await scribe.callModel(transcript); persistObservationDrivenPass({ store, observation: { kind: 'transcript', - source: lastSourceProcessed ?? 'transcript', + source, content: transcript, - metadata: { chunkCount: pendingTranscriptChunks.length }, + metadata, }, artifact: { kind: 'scribe-pass-output', @@ -552,12 +911,11 @@ export function createPluginRegistration(api: PluginApi): PluginRuntimeState { metadata: { stage: 'transcript-scribe' }, }, pass, - source: lastSourceProcessed ?? 'transcript', + source, }); - pendingTranscriptChunks.length = 0; injector.invalidate(); passesSinceReflection++; - status.recordTranscriptScribe(lastSourceProcessed); + status.recordTranscriptScribe(source); status.setNodeLifecycleCounts( store.queryNodes({ active: true }).length, store.queryNodes({ active: false }).length, @@ -570,18 +928,249 @@ export function createPluginRegistration(api: PluginApi): PluginRuntimeState { } }; + const runScribePass = async (): Promise => { + if (pendingTranscriptChunks.length === 0) return 0; + const drainedChunks = pendingTranscriptChunks.splice(0, pendingTranscriptChunks.length); + const transcript = drainedChunks.map((chunk) => chunk.content).join('\n'); + const source = drainedChunks[drainedChunks.length - 1]?.source ?? 'transcript'; + const chunkCount = drainedChunks.length; + + try { + await runTranscriptScribeForSource(source, transcript, { chunkCount }); + } catch (err) { + pendingTranscriptChunks.unshift(...drainedChunks); + throw err; + } + + return 1; + }; + + const indexSessionFiles = async ( + options?: { full?: boolean; limit?: number }, + ): Promise<{ + filesConsidered: number; + unreadCandidates: number; + processed: number; + skipped: number; + failed: number; + remaining: number; + limitApplied: number; + }> => { + if (sessionIndexRunInFlight) { + return sessionIndexRunInFlight; + } + + sessionIndexRunInFlight = (async () => { + const full = options?.full ?? false; + const requestedLimit = options?.limit; + const limitApplied = Math.max( + 1, + requestedLimit ?? (full ? DEFAULT_FULL_REINDEX_LIMIT : DEFAULT_INDEX_SESSION_LIMIT), + ); + const filePaths = new Set(); + const replayPaths = resolveSessionReplayPaths(api, config.watchPaths); + + for (const watchPath of replayPaths) { + const resolvedWatchPath = expandHome(watchPath); + if (!existsSync(resolvedWatchPath)) continue; + if (extname(resolvedWatchPath) === '.jsonl') { + filePaths.add(resolvedWatchPath); + } else { + for (const filePath of scanJsonlFiles(resolvedWatchPath)) { + filePaths.add(filePath); + } + } + } + + const orderedPaths = [...filePaths].sort(); + const cursorSnapshot = watcher.getCursors(); + const pathStates = orderedPaths.map((filePath) => { + const size = watcher.getFileSize(filePath); + const cursor = cursorSnapshot[filePath] ?? 0; + let mtimeMs = 0; + try { + mtimeMs = statSync(filePath).mtimeMs; + } catch { + mtimeMs = 0; + } + return { + filePath, + cursor, + size, + mtimeMs, + hasUnread: size !== null && size > cursor, + }; + }); + const unreadPaths = pathStates + .filter((state) => state.hasUnread) + .sort((a, b) => b.mtimeMs - a.mtimeMs || a.filePath.localeCompare(b.filePath)); + const selectedStates = (full ? pathStates : unreadPaths).slice(0, limitApplied); + let processed = 0; + let skipped = 0; + let failed = 0; + + console.log( + `[memrok] Session indexing started: selected=${selectedStates.length} total=${orderedPaths.length} unread=${unreadPaths.length} mode=${full ? 'full' : 'unread'} limit=${limitApplied}`, + ); + + for (const [index, state] of selectedStates.entries()) { + const filePath = state.filePath; + const previousCursor = full ? 0 : state.cursor; + let result: { content: string | null; nextOffset: number } | null; + try { + result = watcher.readContentFromOffset(filePath, previousCursor); + } catch (err) { + failed++; + status.recordError('session-index', err); + continue; + } + + if (!result) { + skipped++; + continue; + } + + console.log(`[memrok] Session indexing ${index + 1}/${selectedStates.length}: ${filePath}`); + + if (!result.content?.trim()) { + watcher.setCursor(filePath, result.nextOffset); + skipped++; + continue; + } + + try { + await runTranscriptScribeForSource(filePath, result.content, { + stage: full ? 'manual-session-reindex' : 'manual-session-index', + replay: full, + fileIndex: index + 1, + fileCount: selectedStates.length, + }); + watcher.setCursor(filePath, result.nextOffset); + processed++; + } catch (err) { + watcher.setCursor(filePath, previousCursor); + status.recordError('session-index', err); + failed++; + } + } + + watcher.saveCursors(); + + return { + filesConsidered: orderedPaths.length, + unreadCandidates: unreadPaths.length, + processed, + skipped, + failed, + remaining: Math.max((full ? orderedPaths.length : unreadPaths.length) - selectedStates.length, 0), + limitApplied, + }; + })(); + + try { + return await sessionIndexRunInFlight; + } finally { + sessionIndexRunInFlight = null; + } + }; + + const scanMemory = async (force = false) => { + const result = await runBootstrap(store, scribe, config.bootstrap, api, resolveWorkspaceDir(), { force }); + status.setNodeLifecycleCounts( + store.queryNodes({ active: true }).length, + store.queryNodes({ active: false }).length, + ); + return result; + }; + + const flushPendingSessions = async (): Promise => { + const processed = await runScribePass(); + if (processed > 0) { + consolidation.recordPassComplete(); + } + return processed; + }; + watcher.on('data', (filePath: string, content: string) => { - lastSourceProcessed = filePath; - pendingTranscriptChunks.push(content); + pendingTranscriptChunks.push({ source: filePath, content }); const lines = content.split('\n').filter((line) => line.trim()).length; consolidation.recordMessages(lines); }); - consolidation.setTriggerCallback(runScribePass); + consolidation.setTriggerCallback(async () => { + await runScribePass(); + }); - const runtime = { store, injector, watcher, consolidation }; + const runtime: PluginRuntimeState = { + store, + injector, + watcher, + consolidation, + status, + config, + scanMemory, + flushPendingSessions, + indexSessionFiles, + describeBootstrapTargets: () => resolveBootstrapTargets(config.bootstrap, api, resolveWorkspaceDir()), + }; api.registerContextEngine('memrok', () => createContextEngine(runtime)); + api.registerCommand?.({ + name: 'memrok', + nativeNames: { + default: 'memrok', + }, + nativeProgressMessages: { + telegram: 'Memrok is working...', + }, + description: 'Show Memrok status and manually trigger memory/session indexing.', + acceptsArgs: true, + handler: async (ctx) => { + const parsed = parseCommandArgs(ctx.args); + switch (parsed.kind) { + case 'status': { + const activity = status.getStatus(); + const targets = runtime.describeBootstrapTargets(); + return { + text: [ + '**Memrok**', + `db: \`${config.dbPath}\``, + `watch paths: ${config.watchPaths.length}`, + `memory dirs: ${targets.memoryDirs.length}`, + `memory indexes: ${targets.memoryIndexes.length}`, + `active nodes: ${activity.activeNodeCount}`, + `expired nodes: ${activity.expiredNodeCount}`, + `last transcript scribe: ${formatTimestamp(activity.lastTranscriptScribeAt)}`, + `last reflection: ${formatTimestamp(activity.lastReflectiveScribeAt)}`, + `last source: ${activity.lastSourceProcessed ?? 'none'}`, + ].join('\n'), + }; + } + case 'scan-memory': { + const result = await runtime.scanMemory(parsed.force); + return { + text: `Memrok memory scan complete. considered=${result.filesConsidered} processed=${result.processed} skipped=${result.skipped} failed=${result.failed}${parsed.force ? ' force=true' : ''}`, + }; + } + case 'flush-sessions': { + const processed = await runtime.flushPendingSessions(); + return { + text: processed > 0 + ? `Memrok flushed pending session indexing. processed=${processed}` + : 'Memrok has no pending session transcript chunks to flush.', + }; + } + case 'index-sessions': { + const result = await runtime.indexSessionFiles({ full: parsed.full, limit: parsed.limit }); + return { + text: `Memrok session indexing complete. considered=${result.filesConsidered} unread=${result.unreadCandidates} processed=${result.processed} skipped=${result.skipped} failed=${result.failed} remaining=${result.remaining} limit=${result.limitApplied}${parsed.full ? ' mode=full' : ' mode=unread'}`, + }; + } + case 'help': + return { text: buildHelpText(parsed.error) }; + } + }, + }); api.registerService({ id: 'memrok-watcher', async start() { @@ -595,26 +1184,13 @@ export function createPluginRegistration(api: PluginApi): PluginRuntimeState { }, DEFAULT_REFLECTION_CHECK_INTERVAL_MS); await checkAndRunReflection(); - // Auto-bootstrap: seed the graph from existing memory files if no - // non-bootstrap passes have been recorded yet (i.e. fresh graph). if (config.bootstrap.enabled) { - const passes = store.listPasses(); - const hasNonBootstrapPasses = passes.some((p) => p.source && !p.source.startsWith('bootstrap:')); - if (!hasNonBootstrapPasses) { - const workspaceDir = - api.runtime?.agent?.resolveAgentWorkspaceDir?.(api.config) ?? process.cwd(); - runBootstrap(store, scribe, config.bootstrap, workspaceDir).then(() => { - status.setNodeLifecycleCounts( - store.queryNodes({ active: true }).length, - store.queryNodes({ active: false }).length, - ); - }).catch((err) => { - status.recordError('bootstrap', err); - console.warn( - `[memrok] Bootstrap failed: ${err instanceof Error ? err.message : String(err)}`, - ); - }); - } + scanMemory(false).catch((err) => { + status.recordError('bootstrap', err); + console.warn( + `[memrok] Bootstrap failed: ${err instanceof Error ? err.message : String(err)}`, + ); + }); } }, async stop() { diff --git a/packages/openclaw-plugin/src/types.ts b/packages/openclaw-plugin/src/types.ts index 456fd6e..6820edc 100644 --- a/packages/openclaw-plugin/src/types.ts +++ b/packages/openclaw-plugin/src/types.ts @@ -14,12 +14,18 @@ export interface ReflectionConfig { } export interface BootstrapConfig { - /** Enable seeding the graph from existing memory files on first start. Default: true */ + /** Enable seeding the graph from existing memory files. Default: false */ enabled?: boolean; /** Directory to scan for .md files. Default: auto-detect from workspace + '/memory/' */ memoryDir?: string; + /** Additional directories to scan for .md files. */ + memoryDirs?: string[]; /** Path to MEMORY.md index file. Default: auto-detect from workspace + '/MEMORY.md' */ memoryIndex?: string; + /** Additional MEMORY.md index files to scan. */ + memoryIndexes?: string[]; + /** Discover MEMORY.md and memory/ targets from configured OpenClaw agents. Default: true */ + scanConfiguredAgents?: boolean; /** Skip files older than this many days. Default: 90 */ maxAgeDays?: number; /** Delay in ms between processing files to avoid rate limits. Default: 10000 */ @@ -49,7 +55,10 @@ export interface ResolvedReflectionConfig { export interface ResolvedBootstrapConfig { enabled: boolean; memoryDir?: string; + memoryDirs: string[]; memoryIndex?: string; + memoryIndexes: string[]; + scanConfiguredAgents: boolean; maxAgeDays: number; delayMs: number; } @@ -158,6 +167,25 @@ export interface PluginService { stop(): Promise; } +export interface PluginCommandContext { + args?: string; + sessionId?: string; + sessionKey?: string; +} + +export interface PluginCommandResult { + text: string; +} + +export interface PluginCommandDefinition { + name: string; + nativeNames?: Record; + nativeProgressMessages?: Record; + description: string; + acceptsArgs?: boolean; + handler(ctx: PluginCommandContext): Promise; +} + export interface PluginApi { pluginConfig?: Record; config?: unknown; @@ -166,6 +194,7 @@ export interface PluginApi { registrationMode?: 'full' | 'setup-only' | 'setup-runtime'; registerContextEngine(id: string, factory: () => ContextEngine): void; registerService(service: PluginService): void; + registerCommand?(command: PluginCommandDefinition): void; } export interface PluginRegistration { diff --git a/packages/openclaw-plugin/tsup.config.ts b/packages/openclaw-plugin/tsup.config.ts index 1736c57..de6f8b8 100644 --- a/packages/openclaw-plugin/tsup.config.ts +++ b/packages/openclaw-plugin/tsup.config.ts @@ -1,12 +1,42 @@ -export default { +import { copyFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { defineConfig } from 'tsup'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +const workspaceAliases = { + '@memrok/store': resolve(__dirname, '../store/src/index.ts'), + '@memrok/injector': resolve(__dirname, '../injector/src/index.ts'), + '@memrok/scribe': resolve(__dirname, '../scribe/src/index.ts'), + '@memrok/daemon': resolve(__dirname, '../daemon/src/index.ts'), +}; + +export default defineConfig({ entry: ['src/index.ts'], format: ['esm'], - target: 'node18', + target: 'node22', bundle: true, dts: false, outDir: 'dist', - external: ['better-sqlite3', 'chokidar'], - noExternal: [/^@memrok\//], + skipNodeModulesBundle: false, + external: ['node:sqlite'], + noExternal: ['chokidar', /^@memrok\//], clean: true, - onSuccess: 'cp ../scribe/src/system-prompt.md ../scribe/src/reflection-prompt.md dist/', -}; + onSuccess: async () => { + copyFileSync( + resolve(__dirname, '../scribe/src/system-prompt.md'), + resolve(__dirname, 'dist/system-prompt.md'), + ); + copyFileSync( + resolve(__dirname, '../scribe/src/reflection-prompt.md'), + resolve(__dirname, 'dist/reflection-prompt.md'), + ); + }, + esbuildOptions(options) { + options.alias = { + ...(options.alias ?? {}), + ...workspaceAliases, + }; + }, +}); diff --git a/packages/store/TASK.md b/packages/store/TASK.md index cadd56b..aebab20 100644 --- a/packages/store/TASK.md +++ b/packages/store/TASK.md @@ -39,7 +39,7 @@ The store receives scribe pass output — JSON objects with this structure: Implement the store as a TypeScript package with these capabilities: ### 1. Database Setup -- SQLite via `better-sqlite3` +- SQLite via Node's built-in `node:sqlite` - Schema from `~/memrok/docs/graph-schema.md` — implement all tables: `mutations`, `nodes`, `passes`, `embeddings`, `weight_adjustments`, `config`, `schema_version` - Auto-create on first use, migration support via schema_version @@ -94,7 +94,7 @@ Use an in-memory SQLite database (`:memory:`) for tests. ## Constraints - TypeScript, ESM modules (`"type": "module"` in package.json) -- `better-sqlite3` as the only runtime dependency +- No additional runtime dependency for SQLite; use Node's built-in `node:sqlite` - No ORM — raw SQL is fine and preferred for this - Keep it simple — this is a storage layer, not a framework - All timestamps in ISO 8601 UTC diff --git a/packages/store/package.json b/packages/store/package.json index 4f4eb85..04db0ce 100644 --- a/packages/store/package.json +++ b/packages/store/package.json @@ -9,11 +9,7 @@ "build": "tsc", "test": "vitest run" }, - "dependencies": { - "better-sqlite3": "^11.9.1" - }, "devDependencies": { - "@types/better-sqlite3": "^7.6.13", "tsx": "^4.19.4", "typescript": "^5.8.2" } diff --git a/packages/store/src/schema.ts b/packages/store/src/schema.ts index 9290d89..c71da35 100644 --- a/packages/store/src/schema.ts +++ b/packages/store/src/schema.ts @@ -1,4 +1,4 @@ -import type Database from 'better-sqlite3'; +import type { DatabaseSync } from 'node:sqlite'; const CURRENT_VERSION = 4; @@ -158,6 +158,7 @@ CREATE INDEX IF NOT EXISTS idx_working_set_snapshot_items_pass_id ON working_set CREATE INDEX IF NOT EXISTS idx_working_set_snapshot_items_mutation_id ON working_set_snapshot_items(mutation_id); `; +function getCurrentVersion(db: DatabaseSync): number { const SCHEMA_V4 = ` CREATE TABLE IF NOT EXISTS node_hygiene ( node_key TEXT PRIMARY KEY, @@ -206,12 +207,12 @@ function getCurrentVersion(db: Database.Database): number { return row?.version ?? 1; } -function columnExists(db: Database.Database, table: string, column: string): boolean { +function columnExists(db: DatabaseSync, table: string, column: string): boolean { const rows = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>; return rows.some((row) => row.name === column); } -export function initSchema(db: Database.Database): void { +export function initSchema(db: DatabaseSync): void { const hasVersionTable = db.prepare( "SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'" ).get(); diff --git a/packages/store/src/store.ts b/packages/store/src/store.ts index 1e233ac..3ac9984 100644 --- a/packages/store/src/store.ts +++ b/packages/store/src/store.ts @@ -1,4 +1,5 @@ -import Database from 'better-sqlite3'; +import { createRequire } from 'node:module'; +import type { DatabaseSync, SQLInputValue, SQLOutputValue } from 'node:sqlite'; import { initSchema } from './schema.js'; import type { Store, @@ -24,12 +25,34 @@ import type { UpsertNodeHygieneInput, } from './types.js'; +const require = createRequire(import.meta.url); +const nodeSqliteSpecifier = 'node:sqlite'; +const sqliteModule = (process.getBuiltinModule?.('sqlite') as typeof import('node:sqlite') | undefined) + ?? (require(nodeSqliteSpecifier) as typeof import('node:sqlite')); +const { DatabaseSync: DatabaseSyncRuntime } = sqliteModule; + export function createStore(dbPath: string): Store { - const db = new Database(dbPath); - db.pragma('journal_mode = WAL'); - db.pragma('foreign_keys = ON'); + const db = new DatabaseSyncRuntime(dbPath); + db.exec('PRAGMA journal_mode = WAL'); + db.exec('PRAGMA foreign_keys = ON'); initSchema(db); + function withTransaction(fn: () => T): T { + db.exec('BEGIN'); + try { + const result = fn(); + db.exec('COMMIT'); + return result; + } catch (error) { + try { + db.exec('ROLLBACK'); + } catch { + // Ignore rollback errors and preserve the original failure. + } + throw error; + } + } + function parseJson(value: string | null): T | null { if (!value) return null; return JSON.parse(value) as T; @@ -39,7 +62,15 @@ export function createStore(dbPath: string): Store { return value === undefined ? null : JSON.stringify(value); } - function mapArchiveObservation(row: Record | undefined): ArchiveObservation | null { + function asRow(value: unknown): Record | undefined { + return value as Record | undefined; + } + + function asRows(value: unknown): Array> { + return value as Array>; + } + + function mapArchiveObservation(row: Record | undefined): ArchiveObservation | null { if (!row) return null; return { id: row.id as number, @@ -52,7 +83,7 @@ export function createStore(dbPath: string): Store { }; } - function mapDerivedArtifact(row: Record | undefined): DerivedArtifact | null { + function mapDerivedArtifact(row: Record | undefined): DerivedArtifact | null { if (!row) return null; return { id: row.id as number, @@ -64,7 +95,7 @@ export function createStore(dbPath: string): Store { }; } - function mapWorkingSetSnapshot(row: Record | undefined): WorkingSetSnapshot | null { + function mapWorkingSetSnapshot(row: Record | undefined): WorkingSetSnapshot | null { if (!row) return null; return { id: row.id as number, @@ -77,7 +108,7 @@ export function createStore(dbPath: string): Store { }; } - function mapWorkingSetSnapshotItem(row: Record): WorkingSetSnapshotItem { + function mapWorkingSetSnapshotItem(row: Record): WorkingSetSnapshotItem { return { id: row.id as number, snapshot_id: row.snapshot_id as number, @@ -424,7 +455,7 @@ export function createStore(dbPath: string): Store { return 'noop'; } - const applyPassTx = db.transaction((pass: ScribePass): ApplyResult => { + const applyPassTx = (pass: ScribePass): ApplyResult => withTransaction(() => { const timestamp = now(); let nodesCreated = 0; let nodesUpdated = 0; @@ -475,7 +506,7 @@ export function createStore(dbPath: string): Store { function queryNodes(filter?: NodeFilter): Node[] { const conditions: string[] = []; - const params: Record = {}; + const params: Record = {}; const active = filter?.active ?? true; if (active) { @@ -498,6 +529,10 @@ export function createStore(dbPath: string): Store { } const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; + return asRows(db.prepare(`SELECT * FROM nodes ${where} ORDER BY key`).all(params)) as unknown as Node[]; + } + + const createWorkingSetSnapshotTx = ( return db.prepare(`${NODE_WITH_HYGIENE_SELECT} ${where} ORDER BY nodes.key`) .all(params) .map((row) => mapNode(row as Record)!) @@ -550,7 +585,7 @@ export function createStore(dbPath: string): Store { const createWorkingSetSnapshotTx = db.transaction(( input: CreateWorkingSetSnapshotInput, retention?: WorkingSetRetentionPolicy, - ): WorkingSetSnapshotTrace => { + ): WorkingSetSnapshotTrace => withTransaction(() => { const result = insertWorkingSetSnapshot.run({ session_id: input.sessionId ?? null, query: input.query ?? null, @@ -582,10 +617,10 @@ export function createStore(dbPath: string): Store { } } - const snapshot = mapWorkingSetSnapshot(getWorkingSetSnapshotById.get(snapshotId) as Record)!; + const snapshot = mapWorkingSetSnapshot(asRow(getWorkingSetSnapshotById.get(snapshotId)))!; const items = getWorkingSetSnapshotItemsById .all(snapshotId) - .map((row) => mapWorkingSetSnapshotItem(row as Record)); + .map((row: Record) => mapWorkingSetSnapshotItem(row)); return { ...snapshot, items }; }); @@ -596,19 +631,19 @@ export function createStore(dbPath: string): Store { } const artifact = pass.derived_artifact_id - ? mapDerivedArtifact(getDerivedArtifactById.get(pass.derived_artifact_id) as Record) + ? mapDerivedArtifact(asRow(getDerivedArtifactById.get(pass.derived_artifact_id))) : null; const observation = artifact?.observation_id - ? mapArchiveObservation(getArchiveObservationById.get(artifact.observation_id) as Record) + ? mapArchiveObservation(asRow(getArchiveObservationById.get(artifact.observation_id))) : null; return { observation, artifact, pass }; } function rebuild(): void { - const rebuildTx = db.transaction(() => { + withTransaction(() => { db.exec('DELETE FROM nodes'); - const mutations = getAllMutationsOrdered.all() as Mutation[]; + const mutations = asRows(getAllMutationsOrdered.all()) as unknown as Mutation[]; for (const mut of mutations) { applyMutationToNode( { @@ -629,7 +664,6 @@ export function createStore(dbPath: string): Store { ); } }); - rebuildTx(); } return { @@ -642,16 +676,16 @@ export function createStore(dbPath: string): Store { metadata: serializeJson(input.metadata), }); return mapArchiveObservation( - getArchiveObservationById.get(Number(result.lastInsertRowid)) as Record + asRow(getArchiveObservationById.get(Number(result.lastInsertRowid))) )!; }, listArchiveObservations: (limit = 100) => listArchiveObservationRows .all(limit) - .map((row) => mapArchiveObservation(row as Record)!) + .map((row: Record) => mapArchiveObservation(row)!) .filter(Boolean), getArchiveObservation: (id: number) => - mapArchiveObservation(getArchiveObservationById.get(id) as Record), + mapArchiveObservation(asRow(getArchiveObservationById.get(id))), createDerivedArtifact: (input: CreateDerivedArtifactInput) => { const result = insertDerivedArtifact.run({ kind: input.kind, @@ -660,18 +694,21 @@ export function createStore(dbPath: string): Store { metadata: serializeJson(input.metadata), }); return mapDerivedArtifact( - getDerivedArtifactById.get(Number(result.lastInsertRowid)) as Record + asRow(getDerivedArtifactById.get(Number(result.lastInsertRowid))) )!; }, listDerivedArtifacts: (limit = 100) => listDerivedArtifactRows .all(limit) - .map((row) => mapDerivedArtifact(row as Record)!) + .map((row: Record) => mapDerivedArtifact(row)!) .filter(Boolean), getDerivedArtifact: (id: number) => - mapDerivedArtifact(getDerivedArtifactById.get(id) as Record), + mapDerivedArtifact(asRow(getDerivedArtifactById.get(id))), applyPass: (pass: ScribePass) => applyPassTx(pass), queryNodes, + getNode: (key: string) => (asRow(getNodeByKey.get(key)) as unknown as Node) ?? null, + getHistory: (key: string) => asRows(getMutationsByKey.all(key)) as unknown as Mutation[], + listPasses: () => asRows(getAllPasses.all()) as unknown as Pass[], getNode: (key: string) => mapNode(getNodeByKey.get(key) as Record), getHistory: (key: string) => getMutationsByKey.all(key) as Mutation[], getNodeHygiene: (key: string) => @@ -697,19 +734,19 @@ export function createStore(dbPath: string): Store { listWorkingSetSnapshots: (limit = 50) => listWorkingSetSnapshotRows .all(limit) - .map((row) => mapWorkingSetSnapshot(row as Record)!) + .map((row: Record) => mapWorkingSetSnapshot(row)!) .filter(Boolean), getWorkingSetSnapshot: (id: number) => { - const snapshot = mapWorkingSetSnapshot(getWorkingSetSnapshotById.get(id) as Record); + const snapshot = mapWorkingSetSnapshot(asRow(getWorkingSetSnapshotById.get(id))); if (!snapshot) return null; const items = getWorkingSetSnapshotItemsById .all(id) - .map((row) => mapWorkingSetSnapshotItem(row as Record)); + .map((row: Record) => mapWorkingSetSnapshotItem(row)); return { ...snapshot, items }; }, getProvenanceForPass, getProvenanceForWorkingSetSnapshot: (snapshotId: number) => { - const items = getWorkingSetSnapshotItemsById.all(snapshotId) as Array>; + const items = asRows(getWorkingSetSnapshotItemsById.all(snapshotId)); const seenMutations = new Set(); const seenPasses = new Set(); const links: ProvenanceLink[] = [];