diff --git a/CHANGELOG.md b/CHANGELOG.md index d580f3b..f999b08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,29 @@ All notable changes to engram are documented here. Format based on ## [Unreleased] +### Added + +- **`plugin-miner`** — indexes installed Claude Code plugins and their + provided agents as `concept` nodes with `metadata.subkind` of + `"plugin"` or `"agent"`. Plugin-provided skills are also indexed with + `subkind: "skill"`, scored against the project's detected stack, and + linked to their parent plugin via `provided_by` edges. Skills scored + EXTRACTED or INFERRED also get `relevant_to` edges to matching + project files. +- **`config-miner`** — indexes configured hooks (`subkind: "hook"`) and + MCP servers (`subkind: "mcp_server"`) from `~/.claude/settings.json` + and `/.claude/settings.local.json`. Always-on infrastructure + — confidence fixed at EXTRACTED 1.0, no `relevant_to` edges. +- **`stack-detect`** utility (`src/graph/stack-detect.ts`) — pure + function that derives a set of lowercase language/framework tokens + from a snapshot of graph nodes. Single source of truth for stack + detection across miners. +- **New `EdgeRelation` values**: `provided_by` and `relevant_to`. +- Environment variable `ENGRAM_SKIP_ECOSYSTEM=1` disables both new + miners. Also gated behind the existing `options.withSkills` flag so + empty-project and stress tests stay isolated from the real + `~/.claude` tree. + ## [3.0.2] — 2026-04-24 — "MCP Registry" Chore release. No runtime changes. Adds the `mcpName` field to `package.json` @@ -633,8 +656,6 @@ files — existing `.engram/graph.db` files auto-migrate to schema v7. - **HTTP server package.json path resolution** — now resolves correctly from both `src/` (dev) and `dist/` (built) entry points. ---- - ## [0.5.0] — 2026-04-13 — "Context Spine" ### Added diff --git a/docs/superpowers/plans/2026-04-14-engram-ecosystem-miners.md b/docs/superpowers/plans/2026-04-14-engram-ecosystem-miners.md new file mode 100644 index 0000000..e6d7d4f --- /dev/null +++ b/docs/superpowers/plans/2026-04-14-engram-ecosystem-miners.md @@ -0,0 +1,1493 @@ +# Engram Ecosystem Miners — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add two new miners (`plugin-miner`, `config-miner`) and a shared `stack-detect` utility to engram, indexing installed Claude Code plugins, plugin-provided agents, configured hooks, and MCP servers as `concept` nodes with `subkind` discriminators. + +**Architecture:** Two data sources, two miners. `plugin-miner` walks `~/.claude/plugins/` and indexes plugins + their nested agents with stack-based relevance scoring. `config-miner` parses global + project settings JSON and indexes hooks + MCP servers as always-on infrastructure (no scoring). Both miners hook into the existing pipeline in `src/core.ts` after `ast-miner` and `skills-miner`. + +**Tech Stack:** TypeScript, Node.js 20+, Vitest, sql.js-backed GraphStore. No new dependencies. + +**Design spec:** `docs/superpowers/specs/2026-04-14-engram-ecosystem-miners-design.md` + +--- + +## File Structure + +### Create +- `src/graph/stack-detect.ts` — pure function, detects project stack from `GraphNode[]` +- `src/miners/plugin-miner.ts` — plugin + agent indexing with relevance scoring +- `src/miners/config-miner.ts` — hook + MCP server indexing from settings +- `tests/graph/stack-detect.test.ts` +- `tests/plugin-miner.test.ts` +- `tests/config-miner.test.ts` +- `tests/fixtures/claude-dir/installed_plugins.json` + sample plugin tree +- `tests/fixtures/settings/` — sample settings.json and settings.local.json + +### Modify +- `src/graph/schema.ts` — add `provided_by` and `relevant_to` to `EdgeRelation` union +- `src/miners/index.ts` — re-export the two new miners +- `src/core.ts` — invoke both miners in the pipeline, merge their results + +--- + +## Task 1: Extend EdgeRelation schema + +**Files:** +- Modify: `src/graph/schema.ts:46-62` + +- [ ] **Step 1: Read current EdgeRelation union** + +Run: `grep -n "EdgeRelation" src/graph/schema.ts` +Expected: shows the `export type EdgeRelation = ...` union ending with `"triggered_by"`. + +- [ ] **Step 2: Add the two new relations** + +Edit `src/graph/schema.ts`. Change: + +```typescript + // v0.2: skills-miner uses this to link keyword concept nodes to the + // skill concept nodes they activate. Skills themselves use the existing + // `similar_to` relation for cross-references (Related Skills sections). + | "triggered_by"; +``` + +to: + +```typescript + // v0.2: skills-miner uses this to link keyword concept nodes to the + // skill concept nodes they activate. Skills themselves use the existing + // `similar_to` relation for cross-references (Related Skills sections). + | "triggered_by" + // v0.5: ecosystem miners use these to link plugin-provided skills/agents + // to their parent plugin (`provided_by`) and to project files the skill + // is relevant to (`relevant_to`, only emitted for EXTRACTED/INFERRED). + | "provided_by" + | "relevant_to"; +``` + +- [ ] **Step 3: Verify TypeScript still compiles** + +Run: `npm run build` +Expected: build succeeds with no errors. + +- [ ] **Step 4: Commit** + +```bash +git add src/graph/schema.ts +git commit -m "feat(schema): add provided_by and relevant_to edge relations" +``` + +--- + +## Task 2: Stack-detect utility — failing test + +**Files:** +- Create: `tests/graph/stack-detect.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Create `tests/graph/stack-detect.test.ts`: + +```typescript +import { describe, it, expect } from "vitest"; +import { detectStack } from "../../src/graph/stack-detect.js"; +import type { GraphNode } from "../../src/graph/schema.js"; + +function makeFileNode(sourceFile: string, label = sourceFile): GraphNode { + return { + id: `file:${sourceFile}`, + label, + kind: "file", + sourceFile, + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: 0, + queryCount: 0, + metadata: {}, + }; +} + +function makeClassNode(label: string): GraphNode { + return { + id: `class:${label}`, + label, + kind: "class", + sourceFile: "src/x.py", + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: 0, + queryCount: 0, + metadata: {}, + }; +} + +describe("detectStack", () => { + it("returns empty set for empty input", () => { + expect(detectStack([])).toEqual(new Set()); + }); + + it("detects python from .py files", () => { + const nodes = [makeFileNode("src/main.py")]; + expect(detectStack(nodes).has("python")).toBe(true); + }); + + it("detects typescript from .ts and .tsx files", () => { + const nodes = [makeFileNode("src/a.ts"), makeFileNode("src/b.tsx")]; + const stack = detectStack(nodes); + expect(stack.has("typescript")).toBe(true); + }); + + it("detects fastapi framework from class labels", () => { + const nodes = [makeFileNode("src/main.py"), makeClassNode("FastAPIRouter")]; + const stack = detectStack(nodes); + expect(stack.has("python")).toBe(true); + expect(stack.has("fastapi")).toBe(true); + }); + + it("detects mixed stack", () => { + const nodes = [ + makeFileNode("backend/main.py"), + makeFileNode("frontend/app.ts"), + ]; + const stack = detectStack(nodes); + expect(stack.has("python")).toBe(true); + expect(stack.has("typescript")).toBe(true); + }); + + it("ignores non-file non-class nodes for extension detection", () => { + const nodes: GraphNode[] = [ + { + id: "concept:foo", + label: "foo.py", + kind: "concept", + sourceFile: "", + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: 0, + queryCount: 0, + metadata: {}, + }, + ]; + expect(detectStack(nodes).has("python")).toBe(false); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run tests/graph/stack-detect.test.ts` +Expected: all tests FAIL with "Cannot find module '../../src/graph/stack-detect.js'". + +--- + +## Task 3: Stack-detect utility — implementation + +**Files:** +- Create: `src/graph/stack-detect.ts` + +- [ ] **Step 1: Write minimal implementation** + +Create `src/graph/stack-detect.ts`: + +```typescript +/** + * Project stack detection. + * + * Reads a snapshot of graph nodes (typically fresh AST output) and returns + * a set of lowercase tokens describing the project's languages and + * frameworks (e.g. "python", "fastapi", "docker"). Used by ecosystem + * miners to score plugin-provided skills and agents against the current + * project context. + * + * Pure function. No I/O. No store access. Caller provides the nodes. + */ +import type { GraphNode } from "./schema.js"; + +const EXT_TO_LANGUAGE: Record = { + ".py": "python", + ".ts": "typescript", + ".tsx": "typescript", + ".js": "javascript", + ".jsx": "javascript", + ".go": "go", + ".rs": "rust", + ".java": "java", + ".kt": "kotlin", + ".kts": "kotlin", + ".swift": "swift", + ".rb": "ruby", + ".php": "php", + ".c": "c", + ".cpp": "cpp", + ".cc": "cpp", + ".cs": "csharp", + ".pl": "perl", + ".pm": "perl", +}; + +const FRAMEWORK_MARKERS: Record = { + fastapi: "fastapi", + django: "django", + flask: "flask", + pytest: "pytest", + streamlit: "streamlit", + pydantic: "pydantic", + express: "express", + react: "react", + nextjs: "nextjs", + "next.js": "nextjs", + vue: "vue", + angular: "angular", + playwright: "playwright", + gin: "gin", + echo: "echo", + fiber: "fiber", + actix: "actix", + tokio: "tokio", + axum: "axum", + spring: "spring", + springboot: "springboot", + junit: "junit", + docker: "docker", + postgres: "postgres", + postgresql: "postgres", + redis: "redis", + graphql: "graphql", + grpc: "grpc", + duckdb: "duckdb", +}; + +export function detectStack(nodes: readonly GraphNode[]): Set { + const tokens = new Set(); + + for (const node of nodes) { + if (node.kind === "file" && node.sourceFile) { + const ext = node.sourceFile.match(/\.[a-z]+$/i)?.[0]?.toLowerCase(); + if (ext && EXT_TO_LANGUAGE[ext]) { + tokens.add(EXT_TO_LANGUAGE[ext]); + } + } + + if (node.kind === "file" || node.kind === "class" || node.kind === "function") { + const label = node.label.toLowerCase(); + for (const [marker, framework] of Object.entries(FRAMEWORK_MARKERS)) { + if (label.includes(marker)) { + tokens.add(framework); + } + } + } + } + + return tokens; +} +``` + +- [ ] **Step 2: Run tests to verify they pass** + +Run: `npx vitest run tests/graph/stack-detect.test.ts` +Expected: all 6 tests PASS. + +- [ ] **Step 3: Commit** + +```bash +git add src/graph/stack-detect.ts tests/graph/stack-detect.test.ts +git commit -m "feat(graph): add stack-detect utility for project language/framework detection" +``` + +--- + +## Task 4: Relevance scoring — failing test + +**Files:** +- Create: `tests/plugin-miner-scoring.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Create `tests/plugin-miner-scoring.test.ts`: + +```typescript +import { describe, it, expect } from "vitest"; +import { scoreRelevance } from "../src/miners/plugin-miner.js"; + +describe("scoreRelevance", () => { + it("returns INFERRED 0.6 when stack is empty", () => { + const result = scoreRelevance("python-tdd", "Python TDD workflow", new Set()); + expect(result.confidence).toBe("INFERRED"); + expect(result.score).toBe(0.6); + }); + + it("returns EXTRACTED 1.0 when skill name matches stack language", () => { + const stack = new Set(["python"]); + const result = scoreRelevance("python-tdd", "Python TDD workflow", stack); + expect(result.confidence).toBe("EXTRACTED"); + expect(result.score).toBe(1.0); + }); + + it("returns EXTRACTED 1.0 when skill matches stack framework", () => { + const stack = new Set(["python", "fastapi"]); + const result = scoreRelevance("fastapi-patterns", "FastAPI patterns", stack); + expect(result.confidence).toBe("EXTRACTED"); + }); + + it("returns AMBIGUOUS 0.2 when skill mentions non-matching language", () => { + const stack = new Set(["python"]); + const result = scoreRelevance("kotlin-review", "Kotlin code review", stack); + expect(result.confidence).toBe("AMBIGUOUS"); + expect(result.score).toBe(0.2); + }); + + it("returns INFERRED 0.6 for universal keywords when no language mention", () => { + const stack = new Set(["python"]); + const result = scoreRelevance("security-review", "Security audit", stack); + expect(result.confidence).toBe("INFERRED"); + expect(result.score).toBe(0.6); + }); + + it("returns AMBIGUOUS 0.2 when nothing matches", () => { + const stack = new Set(["python"]); + const result = scoreRelevance("random-thing", "unrelated content", stack); + expect(result.confidence).toBe("AMBIGUOUS"); + expect(result.score).toBe(0.2); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run tests/plugin-miner-scoring.test.ts` +Expected: FAIL with "Cannot find module '../src/miners/plugin-miner.js'". + +--- + +## Task 5: Relevance scoring — implementation + +**Files:** +- Create: `src/miners/plugin-miner.ts` (scoring function + stub miner export) + +- [ ] **Step 1: Write minimal implementation** + +Create `src/miners/plugin-miner.ts`: + +```typescript +/** + * Plugin Miner — indexes installed Claude Code plugins, their skills, and + * their agents as concept nodes with subkind discriminators. Scores each + * skill/agent's relevance to the current project stack. + * + * Schema discipline: no new NodeKinds. All new nodes use `kind: "concept"` + * with `metadata.subkind` set to "plugin", "skill", or "agent". Matches + * Nick's skills-miner convention (concept + subkind: "skill"). + * + * Silent failure throughout — malformed plugin installs must not crash + * engram's SessionStart brief. + */ +import type { Confidence, GraphEdge, GraphNode } from "../graph/schema.js"; + +// ─── Relevance scoring ────────────────────────────────────────────────────── + +const LANGUAGE_TOKENS = new Set([ + "python", "typescript", "javascript", "go", "golang", "rust", + "java", "kotlin", "swift", "ruby", "php", "c", "cpp", "csharp", + "perl", "scala", "elixir", "haskell", "lua", "dart", +]); + +const UNIVERSAL_KEYWORDS = new Set([ + "tdd", "test", "testing", "security", "debugging", "debug", + "git", "docker", "deployment", "deploy", "ci", "cd", + "api", "rest", "documentation", "docs", "refactor", + "code-review", "review", "lint", "format", "build", + "verification", "plan", "brainstorm", +]); + +export interface RelevanceScore { + confidence: Confidence; + score: number; +} + +export function scoreRelevance( + name: string, + description: string, + stackTokens: Set +): RelevanceScore { + if (stackTokens.size === 0) { + return { confidence: "INFERRED", score: 0.6 }; + } + + const tokens = `${name} ${description}` + .toLowerCase() + .split(/[\s\-_/.,;:()|]+/) + .filter((t) => t.length > 1); + + let hasLanguageToken = false; + let hasLanguageMatch = false; + + for (const token of tokens) { + if (LANGUAGE_TOKENS.has(token)) { + hasLanguageToken = true; + if (stackTokens.has(token)) { + hasLanguageMatch = true; + } + } + if (stackTokens.has(token)) { + return { confidence: "EXTRACTED", score: 1.0 }; + } + } + + if (hasLanguageToken && !hasLanguageMatch) { + return { confidence: "AMBIGUOUS", score: 0.2 }; + } + + for (const token of tokens) { + if (UNIVERSAL_KEYWORDS.has(token)) { + return { confidence: "INFERRED", score: 0.6 }; + } + } + + return { confidence: "AMBIGUOUS", score: 0.2 }; +} + +// ─── Main miner (stub — filled in Task 7) ─────────────────────────────────── + +export interface PluginMineResult { + nodes: GraphNode[]; + edges: GraphEdge[]; + pluginCount: number; + anomalies: string[]; +} + +export function minePlugins( + _claudeDir: string, + _astNodes: readonly GraphNode[] +): PluginMineResult { + return { nodes: [], edges: [], pluginCount: 0, anomalies: [] }; +} +``` + +- [ ] **Step 2: Run tests to verify they pass** + +Run: `npx vitest run tests/plugin-miner-scoring.test.ts` +Expected: all 6 tests PASS. + +- [ ] **Step 3: Commit** + +```bash +git add src/miners/plugin-miner.ts tests/plugin-miner-scoring.test.ts +git commit -m "feat(plugin-miner): add relevance scoring with stack awareness" +``` + +--- + +## Task 6: Plugin-miner fixture setup + failing tests + +**Files:** +- Create: `tests/fixtures/claude-dir/plugins/installed_plugins.json` +- Create: `tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/skills/tdd/SKILL.md` +- Create: `tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/skills/python-review/SKILL.md` +- Create: `tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/agents/reviewer.md` +- Create: `tests/plugin-miner.test.ts` + +- [ ] **Step 1: Create installed_plugins.json fixture** + +Create `tests/fixtures/claude-dir/plugins/installed_plugins.json`: + +```json +{ + "plugins": { + "sample-plugin@mp": [ + { + "scope": "user", + "installPath": "FIXTURE_ABS_PATH", + "version": "1.2.3", + "installedAt": "2026-04-01T00:00:00Z", + "lastUpdated": "2026-04-10T00:00:00Z", + "gitCommitSha": "abc123" + } + ] + } +} +``` + +Note: `FIXTURE_ABS_PATH` is a sentinel — test code rewrites it at runtime to the absolute path of the fixture plugin directory. + +- [ ] **Step 2: Create fixture SKILL.md files** + +Create `tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/skills/tdd/SKILL.md`: + +```markdown +--- +name: tdd +description: Test-driven development workflow for any project +--- + +Write tests first. +``` + +Create `tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/skills/python-review/SKILL.md`: + +```markdown +--- +name: python-review +description: Python code review with PEP 8 and type-hint checks +--- + +Review Python code. +``` + +- [ ] **Step 3: Create fixture agent** + +Create `tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/agents/reviewer.md`: + +```markdown +--- +name: reviewer +description: General-purpose code review agent +--- + +Review code. +``` + +- [ ] **Step 4: Write the failing tests** + +Create `tests/plugin-miner.test.ts`: + +```typescript +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { mkdirSync, writeFileSync, readFileSync, rmSync, existsSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { minePlugins } from "../src/miners/plugin-miner.js"; +import type { GraphNode } from "../src/graph/schema.js"; + +const FIXTURE_SRC = resolve(__dirname, "fixtures/claude-dir"); + +function setupFixture(): string { + const tmp = join(tmpdir(), `engram-plugin-miner-${Date.now()}-${Math.random()}`); + mkdirSync(tmp, { recursive: true }); + + // Copy fixture tree and rewrite installPath + copyDir(FIXTURE_SRC, tmp); + const manifestPath = join(tmp, "plugins", "installed_plugins.json"); + const manifest = readFileSync(manifestPath, "utf-8"); + const pluginAbs = join(tmp, "plugins", "store", "plugins", "sample-plugin@mp"); + // JSON-escape the path (Windows backslashes) + const escaped = pluginAbs.replace(/\\/g, "\\\\"); + writeFileSync(manifestPath, manifest.replace("FIXTURE_ABS_PATH", escaped)); + return tmp; +} + +function copyDir(src: string, dst: string): void { + const { readdirSync, statSync, copyFileSync } = require("node:fs") as typeof import("node:fs"); + if (!existsSync(dst)) mkdirSync(dst, { recursive: true }); + for (const entry of readdirSync(src)) { + const s = join(src, entry); + const d = join(dst, entry); + if (statSync(s).isDirectory()) copyDir(s, d); + else copyFileSync(s, d); + } +} + +function pyFileNode(path: string): GraphNode { + return { + id: `file:${path}`, + label: path, + kind: "file", + sourceFile: path, + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: 0, + queryCount: 0, + metadata: {}, + }; +} + +describe("minePlugins", () => { + let claudeDir: string; + + beforeAll(() => { + claudeDir = setupFixture(); + }); + + afterAll(() => { + rmSync(claudeDir, { recursive: true, force: true }); + }); + + it("returns empty when claudeDir is missing", () => { + const result = minePlugins("/does/not/exist", []); + expect(result.nodes).toHaveLength(0); + expect(result.edges).toHaveLength(0); + expect(result.pluginCount).toBe(0); + }); + + it("returns empty when ENGRAM_SKIP_ECOSYSTEM=1", () => { + process.env.ENGRAM_SKIP_ECOSYSTEM = "1"; + try { + const result = minePlugins(claudeDir, [pyFileNode("main.py")]); + expect(result.nodes).toHaveLength(0); + expect(result.pluginCount).toBe(0); + } finally { + delete process.env.ENGRAM_SKIP_ECOSYSTEM; + } + }); + + it("indexes plugin, its 2 skills, and 1 agent", () => { + const result = minePlugins(claudeDir, [pyFileNode("main.py")]); + expect(result.pluginCount).toBe(1); + const pluginNodes = result.nodes.filter((n) => n.metadata.subkind === "plugin"); + const skillNodes = result.nodes.filter((n) => n.metadata.subkind === "skill"); + const agentNodes = result.nodes.filter((n) => n.metadata.subkind === "agent"); + expect(pluginNodes).toHaveLength(1); + expect(skillNodes).toHaveLength(2); + expect(agentNodes).toHaveLength(1); + }); + + it("creates provided_by edges from skill/agent to plugin", () => { + const result = minePlugins(claudeDir, [pyFileNode("main.py")]); + const providedBy = result.edges.filter((e) => e.relation === "provided_by"); + expect(providedBy).toHaveLength(3); + for (const e of providedBy) { + expect(e.target).toBe("plugin:sample-plugin"); + } + }); + + it("scores python-review as EXTRACTED when project has python files", () => { + const result = minePlugins(claudeDir, [pyFileNode("main.py")]); + const pyReview = result.nodes.find((n) => n.label === "python-review"); + expect(pyReview?.confidence).toBe("EXTRACTED"); + }); + + it("creates relevant_to edges only for EXTRACTED or INFERRED skills", () => { + const result = minePlugins(claudeDir, [pyFileNode("main.py")]); + const relevantTo = result.edges.filter((e) => e.relation === "relevant_to"); + for (const e of relevantTo) { + const src = result.nodes.find((n) => n.id === e.source); + expect(src?.confidence).not.toBe("AMBIGUOUS"); + } + }); + + it("handles plugin directory without skills/ or agents/ gracefully", () => { + // Create a second plugin with nothing inside + const emptyPluginDir = join(claudeDir, "plugins", "store", "plugins", "empty-plugin@mp"); + mkdirSync(emptyPluginDir, { recursive: true }); + const manifestPath = join(claudeDir, "plugins", "installed_plugins.json"); + const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")); + manifest.plugins["empty-plugin@mp"] = [ + { + scope: "user", + installPath: emptyPluginDir, + version: "0.1.0", + installedAt: "2026-04-01T00:00:00Z", + lastUpdated: "2026-04-01T00:00:00Z", + gitCommitSha: "def456", + }, + ]; + writeFileSync(manifestPath, JSON.stringify(manifest)); + + const result = minePlugins(claudeDir, [pyFileNode("main.py")]); + expect(result.pluginCount).toBe(2); + const emptyPluginNode = result.nodes.find((n) => n.id === "plugin:empty-plugin"); + expect(emptyPluginNode).toBeDefined(); + }); +}); +``` + +- [ ] **Step 5: Run tests to verify they fail** + +Run: `npx vitest run tests/plugin-miner.test.ts` +Expected: FAIL. The "empty dir" and env-var tests may pass (stub returns empty), but the fixture-based tests FAIL because `minePlugins` is a stub returning empty. + +--- + +## Task 7: Plugin-miner implementation + +**Files:** +- Modify: `src/miners/plugin-miner.ts` (replace stub with real implementation) + +- [ ] **Step 1: Replace the stub with the full implementation** + +Replace the `minePlugins` stub at the bottom of `src/miners/plugin-miner.ts` with: + +```typescript +import { + existsSync, + readFileSync, + readdirSync, + statSync, +} from "node:fs"; +import { basename, join } from "node:path"; +import { detectStack } from "../graph/stack-detect.js"; +import { toPosixPath } from "../graph/path-utils.js"; + +const EXT_TO_LANGUAGE: Record = { + ".py": "python", ".ts": "typescript", ".tsx": "typescript", + ".js": "javascript", ".jsx": "javascript", ".go": "go", + ".rs": "rust", ".java": "java", ".kt": "kotlin", + ".swift": "swift", ".rb": "ruby", ".php": "php", +}; + +interface PluginEntry { + scope?: string; + installPath?: string; + version?: string; + installedAt?: string; + lastUpdated?: string; + gitCommitSha?: string; +} + +function parseFrontmatter(content: string): Record { + const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/); + if (!match) return {}; + const result: Record = {}; + for (const line of match[1].replace(/\r/g, "").split("\n")) { + const kv = line.match(/^([A-Za-z_][A-Za-z0-9_-]*)\s*:\s*"?(.*?)"?\s*$/); + if (kv) result[kv[1]] = kv[2]; + } + return result; +} + +function pluginShortName(pluginKey: string): string { + const atIdx = pluginKey.indexOf("@"); + return atIdx > 0 ? pluginKey.slice(0, atIdx) : pluginKey; +} + +function marketplaceName(pluginKey: string): string { + const atIdx = pluginKey.indexOf("@"); + return atIdx > 0 ? pluginKey.slice(atIdx + 1) : "unknown"; +} + +export function minePlugins( + claudeDir: string, + astNodes: readonly GraphNode[] +): PluginMineResult { + const result: PluginMineResult = { nodes: [], edges: [], pluginCount: 0, anomalies: [] }; + + if (process.env.ENGRAM_SKIP_ECOSYSTEM === "1") return result; + if (!existsSync(claudeDir)) return result; + + const manifestPath = join(claudeDir, "plugins", "installed_plugins.json"); + if (!existsSync(manifestPath)) return result; + + let manifest: { plugins?: Record }; + try { + manifest = JSON.parse(readFileSync(manifestPath, "utf-8")); + } catch { + return result; + } + + const pluginEntries = manifest.plugins; + if (!pluginEntries || typeof pluginEntries !== "object") return result; + + const stackTokens = detectStack(astNodes); + const now = Date.now(); + + // Pick first file whose language matches — used as relevant_to target. + const filesByLang = new Map(); + for (const node of astNodes) { + if (node.kind !== "file") continue; + const ext = node.sourceFile.match(/\.[a-z]+$/i)?.[0]?.toLowerCase(); + if (!ext) continue; + const lang = EXT_TO_LANGUAGE[ext]; + if (lang && !filesByLang.has(lang)) filesByLang.set(lang, node); + } + + for (const [pluginKey, entries] of Object.entries(pluginEntries)) { + if (!Array.isArray(entries) || entries.length === 0) continue; + const entry = entries[0]; + if (!entry.installPath || !existsSync(entry.installPath)) continue; + + const pluginName = pluginShortName(pluginKey); + const marketplace = marketplaceName(pluginKey); + const pluginId = `plugin:${pluginName}`; + + result.nodes.push({ + id: pluginId, + label: pluginName, + kind: "concept", + sourceFile: toPosixPath(entry.installPath), + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: now, + queryCount: 0, + metadata: { + miner: "plugin-miner", + subkind: "plugin", + marketplace, + version: entry.version ?? "unknown", + }, + }); + result.pluginCount++; + + // Skills + const skillsDir = join(entry.installPath, "skills"); + if (existsSync(skillsDir)) { + let skillDirs: string[] = []; + try { skillDirs = readdirSync(skillsDir); } catch { skillDirs = []; } + + for (const skillDir of skillDirs) { + if (skillDir.startsWith("temp_git_") || skillDir.startsWith(".")) continue; + const skillPath = join(skillsDir, skillDir); + try { if (!statSync(skillPath).isDirectory()) continue; } catch { continue; } + + const skillMdPath = join(skillPath, "SKILL.md"); + if (!existsSync(skillMdPath)) continue; + + let content: string; + try { content = readFileSync(skillMdPath, "utf-8"); } catch { + result.anomalies.push(skillMdPath); + continue; + } + const fm = parseFrontmatter(content); + const name = fm.name || skillDir; + const description = fm.description || ""; + const { confidence, score } = scoreRelevance(name, description, stackTokens); + const skillId = `skill:${pluginName}/${name}`; + + result.nodes.push({ + id: skillId, + label: name, + kind: "concept", + sourceFile: toPosixPath(skillMdPath), + sourceLocation: null, + confidence, + confidenceScore: score, + lastVerified: now, + queryCount: 0, + metadata: { + miner: "plugin-miner", + subkind: "skill", + description, + sourcePlugin: pluginName, + marketplace, + version: entry.version ?? "unknown", + }, + }); + + result.edges.push({ + source: skillId, + target: pluginId, + relation: "provided_by", + confidence: "EXTRACTED", + confidenceScore: 1.0, + sourceFile: toPosixPath(skillMdPath), + sourceLocation: null, + lastVerified: now, + metadata: { miner: "plugin-miner" }, + }); + + // relevant_to edge for non-AMBIGUOUS skills + if (confidence !== "AMBIGUOUS") { + const lowered = `${name} ${description}`.toLowerCase(); + for (const [lang, fileNode] of filesByLang) { + if (lowered.includes(lang) || confidence === "INFERRED") { + result.edges.push({ + source: skillId, + target: fileNode.id, + relation: "relevant_to", + confidence, + confidenceScore: score, + sourceFile: toPosixPath(skillMdPath), + sourceLocation: null, + lastVerified: now, + metadata: { miner: "plugin-miner", language: lang }, + }); + break; + } + } + } + } + } + + // Agents + const agentsDir = join(entry.installPath, "agents"); + if (existsSync(agentsDir)) { + let agentFiles: string[] = []; + try { agentFiles = readdirSync(agentsDir); } catch { agentFiles = []; } + + for (const agentFile of agentFiles) { + if (!agentFile.endsWith(".md")) continue; + const agentPath = join(agentsDir, agentFile); + try { if (!statSync(agentPath).isFile()) continue; } catch { continue; } + + let content: string; + try { content = readFileSync(agentPath, "utf-8"); } catch { + result.anomalies.push(agentPath); + continue; + } + const fm = parseFrontmatter(content); + const name = fm.name || basename(agentFile, ".md"); + const description = fm.description || ""; + const { confidence, score } = scoreRelevance(name, description, stackTokens); + const agentId = `agent:${pluginName}/${name}`; + + result.nodes.push({ + id: agentId, + label: name, + kind: "concept", + sourceFile: toPosixPath(agentPath), + sourceLocation: null, + confidence, + confidenceScore: score, + lastVerified: now, + queryCount: 0, + metadata: { + miner: "plugin-miner", + subkind: "agent", + description, + sourcePlugin: pluginName, + marketplace, + }, + }); + + result.edges.push({ + source: agentId, + target: pluginId, + relation: "provided_by", + confidence: "EXTRACTED", + confidenceScore: 1.0, + sourceFile: toPosixPath(agentPath), + sourceLocation: null, + lastVerified: now, + metadata: { miner: "plugin-miner" }, + }); + } + } + } + + return result; +} +``` + +Also add the import of `GraphNode` at the top of the file: + +```typescript +import type { Confidence, GraphEdge, GraphNode } from "../graph/schema.js"; +``` + +(It's already imported via the stub signature — verify the import is present.) + +- [ ] **Step 2: Run tests to verify they pass** + +Run: `npx vitest run tests/plugin-miner.test.ts` +Expected: all 7 tests PASS. + +- [ ] **Step 3: Run the full suite to confirm no regressions** + +Run: `npm test` +Expected: all 486+ existing tests still pass, plus the new ones. + +- [ ] **Step 4: Commit** + +```bash +git add src/miners/plugin-miner.ts tests/plugin-miner.test.ts tests/fixtures/claude-dir/ +git commit -m "feat(plugin-miner): index plugins, skills, and agents with relevance scoring" +``` + +--- + +## Task 8: Config-miner fixture setup + failing tests + +**Files:** +- Create: `tests/fixtures/settings/settings.json` +- Create: `tests/fixtures/settings/settings.local.json` +- Create: `tests/config-miner.test.ts` + +- [ ] **Step 1: Create fixture settings files** + +Create `tests/fixtures/settings/settings.json`: + +```json +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup", + "hooks": [ + { "type": "command", "command": "engram intercept" } + ] + } + ], + "PreToolUse": [ + { + "matcher": "*", + "hooks": [ + { "type": "command", "command": "engram intercept" } + ] + } + ] + }, + "mcpServers": { + "context7": { + "command": "npx", + "args": ["-y", "@context7/mcp"] + }, + "playwright": { + "command": "npx", + "args": ["-y", "@playwright/mcp"] + } + } +} +``` + +Create `tests/fixtures/settings/settings.local.json`: + +```json +{ + "hooks": { + "UserPromptSubmit": [ + { + "matcher": "*", + "hooks": [ + { "type": "command", "command": "local-hook.sh" } + ] + } + ] + } +} +``` + +- [ ] **Step 2: Write the failing tests** + +Create `tests/config-miner.test.ts`: + +```typescript +import { describe, it, expect } from "vitest"; +import { resolve } from "node:path"; +import { mineConfig } from "../src/miners/config-miner.js"; + +const FIXTURE_DIR = resolve(__dirname, "fixtures/settings"); +const GLOBAL = resolve(FIXTURE_DIR, "settings.json"); +const LOCAL = resolve(FIXTURE_DIR, "settings.local.json"); + +describe("mineConfig", () => { + it("returns empty when both files are missing", () => { + const result = mineConfig("/nope/settings.json", "/nope/local.json"); + expect(result.nodes).toHaveLength(0); + }); + + it("returns empty when ENGRAM_SKIP_ECOSYSTEM=1", () => { + process.env.ENGRAM_SKIP_ECOSYSTEM = "1"; + try { + const result = mineConfig(GLOBAL, LOCAL); + expect(result.nodes).toHaveLength(0); + } finally { + delete process.env.ENGRAM_SKIP_ECOSYSTEM; + } + }); + + it("indexes hooks from global settings", () => { + const result = mineConfig(GLOBAL, undefined); + const hooks = result.nodes.filter((n) => n.metadata.subkind === "hook"); + expect(hooks.length).toBeGreaterThanOrEqual(2); + const labels = hooks.map((h) => h.label); + expect(labels).toContain("SessionStart:startup"); + expect(labels).toContain("PreToolUse:*"); + }); + + it("indexes MCP servers from global settings only", () => { + const result = mineConfig(GLOBAL, LOCAL); + const mcps = result.nodes.filter((n) => n.metadata.subkind === "mcp_server"); + expect(mcps).toHaveLength(2); + const names = mcps.map((m) => m.label); + expect(names).toContain("context7"); + expect(names).toContain("playwright"); + }); + + it("merges hooks from global and local settings", () => { + const result = mineConfig(GLOBAL, LOCAL); + const hooks = result.nodes.filter((n) => n.metadata.subkind === "hook"); + const labels = hooks.map((h) => h.label); + expect(labels).toContain("UserPromptSubmit:*"); + }); + + it("all hook and mcp nodes have confidence 1.0", () => { + const result = mineConfig(GLOBAL, LOCAL); + for (const n of result.nodes) { + expect(n.confidence).toBe("EXTRACTED"); + expect(n.confidenceScore).toBe(1.0); + } + }); + + it("stores hook command in metadata", () => { + const result = mineConfig(GLOBAL, undefined); + const sessionStart = result.nodes.find((n) => n.label === "SessionStart:startup"); + expect(sessionStart?.metadata.command).toBe("engram intercept"); + }); + + it("handles malformed JSON silently", () => { + // Uses a known-bad path approach: pass a directory as if it were a file + const result = mineConfig(FIXTURE_DIR, undefined); + expect(result.nodes).toHaveLength(0); + }); +}); +``` + +- [ ] **Step 3: Run tests to verify they fail** + +Run: `npx vitest run tests/config-miner.test.ts` +Expected: FAIL with "Cannot find module '../src/miners/config-miner.js'". + +--- + +## Task 9: Config-miner implementation + +**Files:** +- Create: `src/miners/config-miner.ts` + +- [ ] **Step 1: Write implementation** + +Create `src/miners/config-miner.ts`: + +```typescript +/** + * Config Miner — indexes configured Claude Code hooks and MCP servers as + * concept nodes. Always-on infrastructure: no relevance scoring, confidence + * fixed at EXTRACTED 1.0. + * + * Hooks can be configured at global (~/.claude/settings.json) or project + * (.claude/settings.local.json) scope; both are merged. MCP servers only + * come from global settings (matching Claude Code's precedence). + * + * Silent failure throughout — malformed settings must not crash engram. + */ +import { existsSync, readFileSync, statSync } from "node:fs"; +import type { GraphEdge, GraphNode } from "../graph/schema.js"; +import { toPosixPath } from "../graph/path-utils.js"; + +export interface ConfigMineResult { + nodes: GraphNode[]; + edges: GraphEdge[]; +} + +interface HookEntry { + type?: string; + command?: string; +} + +interface HookGroup { + matcher?: string; + hooks?: HookEntry[]; +} + +interface McpServer { + command?: string; + args?: string[]; +} + +interface Settings { + hooks?: Record; + mcpServers?: Record; +} + +function readSettings(path: string | undefined): Settings | null { + if (!path || !existsSync(path)) return null; + try { + if (!statSync(path).isFile()) return null; + } catch { + return null; + } + try { + return JSON.parse(readFileSync(path, "utf-8")) as Settings; + } catch { + return null; + } +} + +export function mineConfig( + globalSettingsPath: string | undefined, + localSettingsPath: string | undefined +): ConfigMineResult { + const result: ConfigMineResult = { nodes: [], edges: [] }; + if (process.env.ENGRAM_SKIP_ECOSYSTEM === "1") return result; + + const global = readSettings(globalSettingsPath); + const local = readSettings(localSettingsPath); + if (!global && !local) return result; + + const now = Date.now(); + const seenHookIds = new Set(); + + for (const [source, settings] of [ + ["global", global], + ["local", local], + ] as const) { + if (!settings?.hooks) continue; + const sourcePath = source === "global" ? globalSettingsPath! : localSettingsPath!; + for (const [hookType, groups] of Object.entries(settings.hooks)) { + if (!Array.isArray(groups)) continue; + for (const group of groups) { + const matcher = group.matcher ?? "*"; + const hooks = Array.isArray(group.hooks) ? group.hooks : []; + for (const h of hooks) { + if (h.type !== "command" || !h.command) continue; + const id = `hook:${hookType}:${matcher}:${h.command}`; + if (seenHookIds.has(id)) continue; + seenHookIds.add(id); + + result.nodes.push({ + id, + label: `${hookType}:${matcher}`, + kind: "concept", + sourceFile: toPosixPath(sourcePath), + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: now, + queryCount: 0, + metadata: { + miner: "config-miner", + subkind: "hook", + hookType, + matcher, + command: h.command, + scope: source, + }, + }); + } + } + } + } + + // MCP servers: global only + if (global?.mcpServers && globalSettingsPath) { + for (const [name, cfg] of Object.entries(global.mcpServers)) { + result.nodes.push({ + id: `mcp:${name}`, + label: name, + kind: "concept", + sourceFile: toPosixPath(globalSettingsPath), + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: now, + queryCount: 0, + metadata: { + miner: "config-miner", + subkind: "mcp_server", + command: cfg?.command ?? "", + args: Array.isArray(cfg?.args) ? cfg.args : [], + }, + }); + } + } + + return result; +} +``` + +- [ ] **Step 2: Run tests to verify they pass** + +Run: `npx vitest run tests/config-miner.test.ts` +Expected: all 8 tests PASS. + +- [ ] **Step 3: Commit** + +```bash +git add src/miners/config-miner.ts tests/config-miner.test.ts tests/fixtures/settings/ +git commit -m "feat(config-miner): index hooks and MCP servers from Claude Code settings" +``` + +--- + +## Task 10: Export new miners from index.ts + +**Files:** +- Modify: `src/miners/index.ts` + +- [ ] **Step 1: Read current index** + +Run: `cat src/miners/index.ts` +Expected output shows 3 re-exports (ast-miner, git-miner, session-miner). Note: skills-miner is deliberately NOT re-exported here (it's imported directly by core.ts). Follow the same pattern for the new miners — they're imported directly by core.ts, no re-export needed. + +- [ ] **Step 2: Decide — no changes to index.ts** + +Since skills-miner is imported directly by core.ts and not re-exported, our new miners follow the same pattern. No edit to `index.ts` required. Skip to Task 11. + +--- + +## Task 11: Wire miners into core.ts pipeline — failing integration test + +**Files:** +- Modify: `tests/core.test.ts` (add one integration test) + +- [ ] **Step 1: Read current core tests** + +Run: `grep -n "describe\|it(" tests/core.test.ts | head -20` +Expected: shows existing test structure. + +- [ ] **Step 2: Add integration test at the end of tests/core.test.ts** + +Append to `tests/core.test.ts`: + +```typescript +import { mineConfig } from "../src/miners/config-miner.js"; +import { minePlugins } from "../src/miners/plugin-miner.js"; + +describe("ecosystem miners integration", () => { + it("plugin-miner and config-miner are invokable with no-op inputs", () => { + // Smoke test — real fixture-based tests live in plugin-miner.test.ts + // and config-miner.test.ts. This just verifies the exports exist and + // are callable from core.ts's perspective. + const pluginResult = minePlugins("/nonexistent", []); + const configResult = mineConfig(undefined, undefined); + expect(pluginResult.nodes).toHaveLength(0); + expect(configResult.nodes).toHaveLength(0); + }); +}); +``` + +- [ ] **Step 3: Run to verify it passes (baseline)** + +Run: `npx vitest run tests/core.test.ts -t "ecosystem miners"` +Expected: PASS (miners already exist, stubs return empty for missing inputs). + +--- + +## Task 12: Wire miners into core.ts pipeline + +**Files:** +- Modify: `src/core.ts:14` (imports), `src/core.ts:75-110` (pipeline) + +- [ ] **Step 1: Add imports** + +Edit `src/core.ts` around line 14 (after the existing skills-miner import). Add: + +```typescript +import { minePlugins } from "./miners/plugin-miner.js"; +import { mineConfig } from "./miners/config-miner.js"; +``` + +- [ ] **Step 2: Invoke miners after skills-miner** + +Find the block in `src/core.ts` that reads: + +```typescript + const allNodes = [ + ...nodes, + ...gitResult.nodes, + ...sessionResult.nodes, + ...skillNodes, + ]; + const allEdges = [ + ...edges, + ...gitResult.edges, + ...sessionResult.edges, + ...skillEdges, + ]; +``` + +Replace with: + +```typescript + // Ecosystem indexing: plugins + nested agents, plus hooks + MCP servers + // from Claude Code settings. Silent failure — these miners never throw. + const claudeDir = DEFAULT_SKILLS_DIR.replace(/[/\\]skills$/, ""); + const pluginResult = minePlugins(claudeDir, nodes); + const globalSettings = join(claudeDir, "settings.json"); + const localSettings = join(root, ".claude", "settings.local.json"); + const configResult = mineConfig(globalSettings, localSettings); + + const allNodes = [ + ...nodes, + ...gitResult.nodes, + ...sessionResult.nodes, + ...skillNodes, + ...pluginResult.nodes, + ...configResult.nodes, + ]; + const allEdges = [ + ...edges, + ...gitResult.edges, + ...sessionResult.edges, + ...skillEdges, + ...pluginResult.edges, + ...configResult.edges, + ]; +``` + +- [ ] **Step 3: Verify `join` is imported from node:path at the top of core.ts** + +Run: `grep -n "from \"node:path\"" src/core.ts` +Expected: shows an existing `import { ... } from "node:path"` line. If `join` is not in the import list, add it. + +- [ ] **Step 4: Run the full test suite** + +Run: `npm test` +Expected: all tests PASS (existing 486 + the new ones from Tasks 2, 4, 6, 8, 11). + +- [ ] **Step 5: Build** + +Run: `npm run build` +Expected: build succeeds. + +- [ ] **Step 6: Commit** + +```bash +git add src/core.ts tests/core.test.ts +git commit -m "feat(core): wire plugin-miner and config-miner into the mining pipeline" +``` + +--- + +## Task 13: Manual end-to-end verification + +**Files:** +- No code changes — verification only. + +- [ ] **Step 1: Re-mine seo-brain** + +Run from `c:/Users/shahe/seo-brain`: `node c:/Users/shahe/engram/dist/cli.js init` +Expected: command completes without errors. Output should show a higher node count than before (previously 261; should now include plugin/agent/hook/mcp nodes). + +- [ ] **Step 2: Inspect the new nodes** + +Run from `c:/Users/shahe/seo-brain`: + +```bash +sqlite3 .engram/graph.db "SELECT kind, json_extract(metadata, '$.subkind') AS subkind, COUNT(*) FROM nodes GROUP BY kind, subkind;" +``` + +Expected: rows showing `concept | plugin | N`, `concept | skill | N`, `concept | agent | N`, `concept | hook | N`, `concept | mcp_server | N`. + +- [ ] **Step 3: Verify SessionStart brief still works** + +Open a new Claude Code session in seo-brain and confirm the startup brief appears without errors (look for the `[engram] Project brief for seo-brain` line). + +- [ ] **Step 4: Test the escape hatch** + +Run: `ENGRAM_SKIP_ECOSYSTEM=1 node c:/Users/shahe/engram/dist/cli.js init` +Expected: command succeeds, node count is back to pre-ecosystem baseline (no plugin/hook/mcp nodes in the DB). + +- [ ] **Step 5: Commit a CHANGELOG update** + +Edit `CHANGELOG.md`. Add a new entry at the top: + +```markdown +## Unreleased + +### Added +- `plugin-miner`: indexes installed Claude Code plugins and their provided agents as `concept` nodes with `subkind: "plugin"` and `subkind: "agent"`. +- `config-miner`: indexes configured hooks (`subkind: "hook"`) and MCP servers (`subkind: "mcp_server"`) from global and project settings. +- `stack-detect` utility in `src/graph/`: shared language/framework detection from AST nodes. +- New `EdgeRelation` values: `provided_by` and `relevant_to`. +- Environment variable `ENGRAM_SKIP_ECOSYSTEM=1` disables both new miners. +``` + +```bash +git add CHANGELOG.md +git commit -m "docs: changelog entry for ecosystem miners" +``` + +--- + +## Self-review checklist + +- [x] Spec coverage: stack-detect (T2-3), relevance scoring (T4-5), plugin-miner (T6-7), config-miner (T8-9), schema edges (T1), pipeline integration (T11-12), verification (T13). +- [x] No placeholders — every step shows the actual code/command. +- [x] Type consistency: `PluginMineResult` defined in T5, used in T7. `ConfigMineResult` defined in T9. `scoreRelevance` signature consistent across tasks. +- [x] Every test file has both failing-test and passing-impl tasks. +- [x] Windows compatibility: uses `toPosixPath` for sourceFile writes, matches Nick's cross-platform convention. diff --git a/docs/superpowers/specs/2026-04-14-engram-ecosystem-miners-design.md b/docs/superpowers/specs/2026-04-14-engram-ecosystem-miners-design.md new file mode 100644 index 0000000..f9e0078 --- /dev/null +++ b/docs/superpowers/specs/2026-04-14-engram-ecosystem-miners-design.md @@ -0,0 +1,181 @@ +# Engram Ecosystem Miners — Design + +**Date:** 2026-04-14 +**Author:** Shahe +**Status:** Approved, ready for implementation plan +**Target:** engram (fork at shahe-dev/engram, upstream at NickCirv/engram@v0.4.4) + +## Goal + +Extend engram to index the Claude Code plugin ecosystem — plugins, agents, hooks, and MCP servers — so they appear as first-class nodes in the knowledge graph alongside code structure, skills, git history, and sessions. + +Today, engram's `skills-miner` indexes SKILL.md files but ignores the surrounding ecosystem: which plugin owns a skill, what agents a plugin provides, which hooks are configured, and which MCP servers are installed. Adding these closes the gap without duplicating any of Nick's existing mining logic. + +## Non-goals + +- No remote plugin marketplace indexing (local `~/.claude/plugins/` only). +- No plugin version history or update detection. +- No hook command parsing — store the command string, don't try to understand it. +- No MCP server runtime status checks. +- No trigger extraction from plugin skills (Nick's `skills-miner` already handles SKILL.md triggers; we do not duplicate). + +## Architecture + +Two new miners and one shared utility, following Nick's conventions: + +``` +src/graph/ + stack-detect.ts (NEW) — project stack detection from store +src/miners/ + plugin-miner.ts (NEW) — plugins + nested agents + config-miner.ts (NEW) — hooks + MCP servers +``` + +### `src/graph/stack-detect.ts` + +Pure function `detectStack(store: GraphStore): Set`. Reads `file` and `class`/`function` nodes from the store, maps extensions to languages (`.py` → `python`, `.ts` → `typescript`, etc.), and matches labels against framework markers (`fastapi`, `react`, `docker`, etc.). Returns lowercase tokens. + +Single source of truth for stack detection. Future miners that need project-context awareness import from here rather than reinventing. + +### `src/miners/plugin-miner.ts` + +Input: `claudeDir` (default `~/.claude`), `store` (for stack detection). +Output: nodes for each plugin, skill, and agent; edges `provided_by` (skill/agent → plugin) and `relevant_to` (skill/agent → project file) when relevance scores EXTRACTED or INFERRED. + +Reads `plugins/installed_plugins.json`, walks each plugin's `skills/` and `agents/` directories, parses SKILL.md and agent frontmatter. + +### `src/miners/config-miner.ts` + +Input: `settingsPath` (global `~/.claude/settings.json`) and `localSettingsPath` (project `/.claude/settings.local.json`). +Output: nodes for each hook and MCP server. No relevance scoring, no `relevant_to` edges. Confidence 1.0 — these are always-on infrastructure, not recommendations. + +## Schema + +### No new `NodeKind` values + +All new nodes use `kind: "concept"` with a `metadata.subkind` discriminator, following Nick's established pattern (his skills-miner uses `concept + subkind: "skill"`). + +| Entity | `kind` | `metadata.subkind` | id format | +|---|---|---|---| +| Plugin | `concept` | `plugin` | `plugin:` | +| Agent | `concept` | `agent` | `agent:/` | +| Hook | `concept` | `hook` | `hook::` | +| MCP server | `concept` | `mcp_server` | `mcp:` | + +### New `EdgeRelation` values + +Additive to the existing enum in `src/graph/schema.ts`: + +- `provided_by` — skill/agent → plugin +- `relevant_to` — skill/agent → file (only emitted when source node scored EXTRACTED or INFERRED) + +## Data flow + +### Plugin-miner + +1. Read `~/.claude/plugins/installed_plugins.json`. +2. For each plugin entry: create a plugin concept node (`id = plugin:`). +3. For each `SKILL.md` in the plugin's `skills/` directory: + - Parse frontmatter (reuse Nick's YAML parser — extract to a shared helper or import from `skills-miner`). + - Score relevance against stack tokens via `detectStack`. + - Create skill node with confidence from scoring. + - Create `provided_by` edge → plugin. + - If confidence is EXTRACTED or INFERRED, create one `relevant_to` edge to the first matching file node in the store. +4. Same loop for each `.md` in the plugin's `agents/` directory. + +### Config-miner + +1. Read `~/.claude/settings.json` and `/.claude/settings.local.json`. +2. For each hook entry (PreToolUse / PostToolUse / SessionStart / UserPromptSubmit / Stop / etc., plus matcher and command): + - Create hook node with confidence 1.0. +3. For each MCP server in global settings: + - Create MCP node with confidence 1.0. + +Local settings may add hooks; MCP servers only come from global settings (matching Claude Code's own precedence). + +## Relevance scoring + +Applied to skills and agents only. Hooks and MCP servers get confidence 1.0 because they are always-on infrastructure, not LLM-selected suggestions. + +| Match type | Confidence | Score | +|---|---|---| +| Skill token matches a detected language or framework | `EXTRACTED` | 1.0 | +| Skill mentions a universal dev keyword (tdd, debug, security, etc.) | `INFERRED` | 0.6 | +| Skill mentions a language token that does NOT match detected stack | `AMBIGUOUS` | 0.2 | +| No match, no language mismatch | `AMBIGUOUS` | 0.2 | + +Stack set is empty when the store has no AST nodes yet — in that case skills default to `INFERRED` / 0.6 (can't rule anything out yet). + +## Error handling + +Both miners fail silently and gracefully, matching Nick's existing convention. + +- `~/.claude/` missing → return `{nodes: [], edges: []}` +- `installed_plugins.json` missing or malformed → return empty +- Individual plugin's `skills/` or `agents/` directory missing → skip, continue +- Individual `SKILL.md` or agent file unparseable → skip that file, continue (optionally append to an `anomalies[]` array for diagnostics) +- Broken symlinks → skip silently +- Settings JSON malformed → return empty from that source, still process the other + +No exception escapes either miner. A corrupted plugin install must not crash engram's SessionStart brief. + +### Environment escape hatch + +Mirror Nick's `ENGRAM_SKIP_SKILLS` pattern: + +- `ENGRAM_SKIP_ECOSYSTEM=1` → both new miners return empty. Used by CI and tests. + +## Testing + +### `tests/graph/stack-detect.test.ts` + +- Nodes with `.py` files → set includes `"python"`. +- Node labels containing `fastapi` → set includes `"fastapi"`. +- Empty store → returns empty set. +- Mixed stack (Python + TypeScript) → includes both. + +### `tests/plugin-miner.test.ts` + +Fixtures under `tests/fixtures/claude-dir/`. + +- Missing `.claude/` → empty. +- Missing `installed_plugins.json` → empty. +- Malformed JSON → empty. +- Valid plugin with 2 skills and 1 agent → 3 content nodes + 3 `provided_by` edges + 1 plugin node. +- Skill matching stack → EXTRACTED confidence + `relevant_to` edge created. +- Skill not matching stack → AMBIGUOUS confidence, no `relevant_to` edge. +- Plugin with no `skills/` or `agents/` → plugin node only. +- Corrupted `SKILL.md` → skipped, other skills still indexed. +- `ENGRAM_SKIP_ECOSYSTEM=1` → empty. + +### `tests/config-miner.test.ts` + +- Missing settings files → empty. +- Malformed JSON → empty. +- Valid settings with 3 hooks + 2 MCP servers → 5 nodes, all confidence 1.0. +- `mcpServers` without `hooks` → MCP nodes only. +- Local + global settings both provide hooks → both sets indexed. +- `ENGRAM_SKIP_ECOSYSTEM=1` → empty. + +### Integration + +One integration test in `tests/core.test.ts` against a fixture repo, verifying the full pipeline produces expected counts. + +## Pipeline integration + +Miners run in `src/core.ts` in this order: + +1. `ast-miner` — provides the data `stack-detect` reads. +2. `skills-miner` — Nick's existing miner, unchanged. +3. `plugin-miner` — new. +4. `config-miner` — new. + +## Upstream contribution path + +After local implementation and verification, prepare as a single PR to `NickCirv/engram`: + +- One commit per miner (plugin-miner, config-miner, stack-detect) to make review easier. +- PR description explains the gap, the schema discipline (concept+subkind, no new NodeKinds), and the feature-flag escape hatch. +- Tests land with each commit. + +Windows/Node 25 fixes (already upstream as of v0.4.4) are not part of this PR. diff --git a/src/core.ts b/src/core.ts index ec3448f..27bfcaf 100644 --- a/src/core.ts +++ b/src/core.ts @@ -12,6 +12,8 @@ import { extractDirectory } from "./miners/ast-miner.js"; import { mineGitHistory } from "./miners/git-miner.js"; import { mineSessionHistory, learnFromSession } from "./miners/session-miner.js"; import { mineSkills } from "./miners/skills-miner.js"; +import { minePlugins } from "./miners/plugin-miner.js"; +import { mineConfig } from "./miners/config-miner.js"; import type { GraphStats } from "./graph/schema.js"; const ENGRAM_DIR = ".engram"; @@ -116,17 +118,38 @@ export async function init( skillEdges = skillsResult.edges; } + // Ecosystem indexing: plugins + nested agents, plus hooks + MCP servers + // from Claude Code settings. Gated behind the same opt-in flag as the + // skills-miner so stress tests and empty-project tests stay clean. + // Silent failure — these miners never throw. + let pluginResult: { nodes: typeof nodes; edges: typeof edges; pluginCount: number; anomalies: string[] } = + { nodes: [], edges: [], pluginCount: 0, anomalies: [] }; + let configResult: { nodes: typeof nodes; edges: typeof edges } = { nodes: [], edges: [] }; + if (options.withSkills) { + const skillsDirForCalc = + typeof options.withSkills === "string" ? options.withSkills : DEFAULT_SKILLS_DIR; + const claudeDir = skillsDirForCalc.replace(/[/\\]skills$/, ""); + pluginResult = minePlugins(claudeDir, nodes); + const globalSettings = join(claudeDir, "settings.json"); + const localSettings = join(root, ".claude", "settings.local.json"); + configResult = mineConfig(globalSettings, localSettings); + } + const allNodes = [ ...nodes, ...gitResult.nodes, ...sessionResult.nodes, ...skillNodes, + ...pluginResult.nodes, + ...configResult.nodes, ]; const allEdges = [ ...edges, ...gitResult.edges, ...sessionResult.edges, ...skillEdges, + ...pluginResult.edges, + ...configResult.edges, ]; const store = await getStore(root); diff --git a/src/graph/schema.ts b/src/graph/schema.ts index 25e2db5..d000953 100644 --- a/src/graph/schema.ts +++ b/src/graph/schema.ts @@ -72,7 +72,12 @@ export type EdgeRelation = // v0.2: skills-miner uses this to link keyword concept nodes to the // skill concept nodes they activate. Skills themselves use the existing // `similar_to` relation for cross-references (Related Skills sections). - | "triggered_by"; + | "triggered_by" + // v0.5: ecosystem miners use these to link plugin-provided skills/agents + // to their parent plugin (`provided_by`) and to project files the skill + // is relevant to (`relevant_to`, only emitted for EXTRACTED/INFERRED). + | "provided_by" + | "relevant_to"; export interface GraphStats { readonly nodes: number; diff --git a/src/graph/stack-detect.ts b/src/graph/stack-detect.ts new file mode 100644 index 0000000..9954b30 --- /dev/null +++ b/src/graph/stack-detect.ts @@ -0,0 +1,90 @@ +/** + * Project stack detection. + * + * Reads a snapshot of graph nodes (typically fresh AST output) and returns + * a set of lowercase tokens describing the project's languages and + * frameworks (e.g. "python", "fastapi", "docker"). Used by ecosystem + * miners to score plugin-provided skills and agents against the current + * project context. + * + * Pure function. No I/O. No store access. Caller provides the nodes. + */ +import type { GraphNode } from "./schema.js"; + +const EXT_TO_LANGUAGE: Record = { + ".py": "python", + ".ts": "typescript", + ".tsx": "typescript", + ".js": "javascript", + ".jsx": "javascript", + ".go": "go", + ".rs": "rust", + ".java": "java", + ".kt": "kotlin", + ".kts": "kotlin", + ".swift": "swift", + ".rb": "ruby", + ".php": "php", + ".c": "c", + ".cpp": "cpp", + ".cc": "cpp", + ".cs": "csharp", + ".pl": "perl", + ".pm": "perl", +}; + +const FRAMEWORK_MARKERS: Record = { + fastapi: "fastapi", + django: "django", + flask: "flask", + pytest: "pytest", + streamlit: "streamlit", + pydantic: "pydantic", + express: "express", + react: "react", + nextjs: "nextjs", + "next.js": "nextjs", + vue: "vue", + angular: "angular", + playwright: "playwright", + gin: "gin", + echo: "echo", + fiber: "fiber", + actix: "actix", + tokio: "tokio", + axum: "axum", + spring: "spring", + springboot: "springboot", + junit: "junit", + docker: "docker", + postgres: "postgres", + postgresql: "postgres", + redis: "redis", + graphql: "graphql", + grpc: "grpc", + duckdb: "duckdb", +}; + +export function detectStack(nodes: readonly GraphNode[]): Set { + const tokens = new Set(); + + for (const node of nodes) { + if (node.kind === "file" && node.sourceFile) { + const ext = node.sourceFile.match(/\.[a-z]+$/i)?.[0]?.toLowerCase(); + if (ext && EXT_TO_LANGUAGE[ext]) { + tokens.add(EXT_TO_LANGUAGE[ext]); + } + } + + if (node.kind === "file" || node.kind === "class" || node.kind === "function") { + const label = node.label.toLowerCase(); + for (const [marker, framework] of Object.entries(FRAMEWORK_MARKERS)) { + if (label.includes(marker)) { + tokens.add(framework); + } + } + } + } + + return tokens; +} diff --git a/src/miners/config-miner.ts b/src/miners/config-miner.ts new file mode 100644 index 0000000..a054e90 --- /dev/null +++ b/src/miners/config-miner.ts @@ -0,0 +1,133 @@ +/** + * Config Miner - indexes configured Claude Code hooks and MCP servers as + * concept nodes. Always-on infrastructure: no relevance scoring, confidence + * fixed at EXTRACTED 1.0. + * + * Hooks can be configured at global (~/.claude/settings.json) or project + * (.claude/settings.local.json) scope; both are merged. MCP servers only + * come from global settings (matching Claude Code's precedence). + * + * Silent failure throughout - malformed settings must not crash engram. + */ +import { existsSync, readFileSync, statSync } from "node:fs"; +import type { GraphEdge, GraphNode } from "../graph/schema.js"; +import { toPosixPath } from "../graph/path-utils.js"; + +export interface ConfigMineResult { + nodes: GraphNode[]; + edges: GraphEdge[]; +} + +interface HookEntry { + type?: string; + command?: string; +} + +interface HookGroup { + matcher?: string; + hooks?: HookEntry[]; +} + +interface McpServer { + command?: string; + args?: string[]; +} + +interface Settings { + hooks?: Record; + mcpServers?: Record; +} + +function readSettings(path: string | undefined): Settings | null { + if (!path || !existsSync(path)) return null; + try { + if (!statSync(path).isFile()) return null; + } catch { + return null; + } + try { + return JSON.parse(readFileSync(path, "utf-8")) as Settings; + } catch { + return null; + } +} + +export function mineConfig( + globalSettingsPath: string | undefined, + localSettingsPath: string | undefined +): ConfigMineResult { + const result: ConfigMineResult = { nodes: [], edges: [] }; + if (process.env.ENGRAM_SKIP_ECOSYSTEM === "1") return result; + + const global = readSettings(globalSettingsPath); + const local = readSettings(localSettingsPath); + if (!global && !local) return result; + + const now = Date.now(); + const seenHookIds = new Set(); + + for (const [source, settings] of [ + ["global", global], + ["local", local], + ] as const) { + if (!settings?.hooks) continue; + const sourcePath = source === "global" ? globalSettingsPath! : localSettingsPath!; + for (const [hookType, groups] of Object.entries(settings.hooks)) { + if (!Array.isArray(groups)) continue; + for (const group of groups) { + const matcher = group.matcher ?? "*"; + const hooks = Array.isArray(group.hooks) ? group.hooks : []; + for (const h of hooks) { + if (h.type !== "command" || !h.command) continue; + const id = `hook:${hookType}:${matcher}:${h.command}`; + if (seenHookIds.has(id)) continue; + seenHookIds.add(id); + + result.nodes.push({ + id, + label: `${hookType}:${matcher}`, + kind: "concept", + sourceFile: toPosixPath(sourcePath), + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: now, + queryCount: 0, + metadata: { + miner: "config-miner", + subkind: "hook", + hookType, + matcher, + command: h.command, + scope: source, + }, + }); + } + } + } + } + + if (global?.mcpServers && globalSettingsPath) { + for (const [name, cfg] of Object.entries(global.mcpServers)) { + result.nodes.push({ + id: `mcp:${name}`, + label: name, + kind: "concept", + sourceFile: toPosixPath(globalSettingsPath), + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: now, + queryCount: 0, + metadata: { + miner: "config-miner", + subkind: "mcp_server", + command: cfg?.command ?? "", + args: Array.isArray(cfg?.args) ? cfg.args : [], + }, + }); + } + } + + return result; +} diff --git a/src/miners/plugin-miner.ts b/src/miners/plugin-miner.ts new file mode 100644 index 0000000..f96e683 --- /dev/null +++ b/src/miners/plugin-miner.ts @@ -0,0 +1,329 @@ +/** + * Plugin Miner — indexes installed Claude Code plugins, their skills, and + * their agents as concept nodes with subkind discriminators. Scores each + * skill/agent's relevance to the current project stack. + * + * Schema discipline: no new NodeKinds. All new nodes use `kind: "concept"` + * with `metadata.subkind` set to "plugin", "skill", or "agent". Matches + * Nick's skills-miner convention (concept + subkind: "skill"). + * + * Silent failure throughout — malformed plugin installs must not crash + * engram's SessionStart brief. + */ +import type { Confidence, GraphEdge, GraphNode } from "../graph/schema.js"; +import { + existsSync, + readFileSync, + readdirSync, + statSync, +} from "node:fs"; +import { basename, join } from "node:path"; +import { detectStack } from "../graph/stack-detect.js"; +import { toPosixPath } from "../graph/path-utils.js"; + +// ─── Relevance scoring ────────────────────────────────────────────────────── + +const LANGUAGE_TOKENS = new Set([ + "python", "typescript", "javascript", "go", "golang", "rust", + "java", "kotlin", "swift", "ruby", "php", "c", "cpp", "csharp", + "perl", "scala", "elixir", "haskell", "lua", "dart", +]); + +const UNIVERSAL_KEYWORDS = new Set([ + "tdd", "test", "testing", "security", "debugging", "debug", + "git", "docker", "deployment", "deploy", "ci", "cd", + "api", "rest", "documentation", "docs", "refactor", + "code-review", "review", "lint", "format", "build", + "verification", "plan", "brainstorm", +]); + +export interface RelevanceScore { + confidence: Confidence; + score: number; +} + +export function scoreRelevance( + name: string, + description: string, + stackTokens: Set +): RelevanceScore { + if (stackTokens.size === 0) { + return { confidence: "INFERRED", score: 0.6 }; + } + + const tokens = `${name} ${description}` + .toLowerCase() + .split(/[\s\-_/.,;:()|]+/) + .filter((t) => t.length > 1); + + let hasLanguageToken = false; + let hasLanguageMatch = false; + + for (const token of tokens) { + if (LANGUAGE_TOKENS.has(token)) { + hasLanguageToken = true; + if (stackTokens.has(token)) { + hasLanguageMatch = true; + } + } + if (stackTokens.has(token)) { + return { confidence: "EXTRACTED", score: 1.0 }; + } + } + + if (hasLanguageToken && !hasLanguageMatch) { + return { confidence: "AMBIGUOUS", score: 0.2 }; + } + + for (const token of tokens) { + if (UNIVERSAL_KEYWORDS.has(token)) { + return { confidence: "INFERRED", score: 0.6 }; + } + } + + return { confidence: "AMBIGUOUS", score: 0.2 }; +} + +// ─── Main miner (stub — filled in Task 7) ─────────────────────────────────── + +export interface PluginMineResult { + nodes: GraphNode[]; + edges: GraphEdge[]; + pluginCount: number; + anomalies: string[]; +} + +const EXT_TO_LANGUAGE: Record = { + ".py": "python", ".ts": "typescript", ".tsx": "typescript", + ".js": "javascript", ".jsx": "javascript", ".go": "go", + ".rs": "rust", ".java": "java", ".kt": "kotlin", + ".swift": "swift", ".rb": "ruby", ".php": "php", +}; + +interface PluginEntry { + scope?: string; + installPath?: string; + version?: string; + installedAt?: string; + lastUpdated?: string; + gitCommitSha?: string; +} + +function parseFrontmatter(content: string): Record { + const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/); + if (!match) return {}; + const result: Record = {}; + for (const line of match[1].replace(/\r/g, "").split("\n")) { + const kv = line.match(/^([A-Za-z_][A-Za-z0-9_-]*)\s*:\s*"?(.*?)"?\s*$/); + if (kv) result[kv[1]] = kv[2]; + } + return result; +} + +function pluginShortName(pluginKey: string): string { + const atIdx = pluginKey.indexOf("@"); + return atIdx > 0 ? pluginKey.slice(0, atIdx) : pluginKey; +} + +function marketplaceName(pluginKey: string): string { + const atIdx = pluginKey.indexOf("@"); + return atIdx > 0 ? pluginKey.slice(atIdx + 1) : "unknown"; +} + +export function minePlugins( + claudeDir: string, + astNodes: readonly GraphNode[] +): PluginMineResult { + const result: PluginMineResult = { nodes: [], edges: [], pluginCount: 0, anomalies: [] }; + + if (process.env.ENGRAM_SKIP_ECOSYSTEM === "1") return result; + if (!existsSync(claudeDir)) return result; + + const manifestPath = join(claudeDir, "plugins", "installed_plugins.json"); + if (!existsSync(manifestPath)) return result; + + let manifest: { plugins?: Record }; + try { + manifest = JSON.parse(readFileSync(manifestPath, "utf-8")); + } catch { + return result; + } + + const pluginEntries = manifest.plugins; + if (!pluginEntries || typeof pluginEntries !== "object") return result; + + const stackTokens = detectStack(astNodes); + const now = Date.now(); + + const filesByLang = new Map(); + for (const node of astNodes) { + if (node.kind !== "file") continue; + const ext = node.sourceFile.match(/\.[a-z]+$/i)?.[0]?.toLowerCase(); + if (!ext) continue; + const lang = EXT_TO_LANGUAGE[ext]; + if (lang && !filesByLang.has(lang)) filesByLang.set(lang, node); + } + + for (const [pluginKey, entries] of Object.entries(pluginEntries)) { + if (!Array.isArray(entries) || entries.length === 0) continue; + const entry = entries[0]; + if (!entry.installPath || !existsSync(entry.installPath)) continue; + + const pluginName = pluginShortName(pluginKey); + const marketplace = marketplaceName(pluginKey); + const pluginId = `plugin:${pluginName}`; + + result.nodes.push({ + id: pluginId, + label: pluginName, + kind: "concept", + sourceFile: toPosixPath(entry.installPath), + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: now, + queryCount: 0, + metadata: { + miner: "plugin-miner", + subkind: "plugin", + marketplace, + version: entry.version ?? "unknown", + }, + }); + result.pluginCount++; + + const skillsDir = join(entry.installPath, "skills"); + if (existsSync(skillsDir)) { + let skillDirs: string[] = []; + try { skillDirs = readdirSync(skillsDir); } catch { skillDirs = []; } + + for (const skillDir of skillDirs) { + if (skillDir.startsWith("temp_git_") || skillDir.startsWith(".")) continue; + const skillPath = join(skillsDir, skillDir); + try { if (!statSync(skillPath).isDirectory()) continue; } catch { continue; } + + const skillMdPath = join(skillPath, "SKILL.md"); + if (!existsSync(skillMdPath)) continue; + + let content: string; + try { content = readFileSync(skillMdPath, "utf-8"); } catch { + result.anomalies.push(skillMdPath); + continue; + } + const fm = parseFrontmatter(content); + const name = fm.name || skillDir; + const description = fm.description || ""; + const { confidence, score } = scoreRelevance(name, description, stackTokens); + const skillId = `skill:${pluginName}/${name}`; + + result.nodes.push({ + id: skillId, + label: name, + kind: "concept", + sourceFile: toPosixPath(skillMdPath), + sourceLocation: null, + confidence, + confidenceScore: score, + lastVerified: now, + queryCount: 0, + metadata: { + miner: "plugin-miner", + subkind: "skill", + description, + sourcePlugin: pluginName, + marketplace, + version: entry.version ?? "unknown", + }, + }); + + result.edges.push({ + source: skillId, + target: pluginId, + relation: "provided_by", + confidence: "EXTRACTED", + confidenceScore: 1.0, + sourceFile: toPosixPath(skillMdPath), + sourceLocation: null, + lastVerified: now, + metadata: { miner: "plugin-miner" }, + }); + + if (confidence !== "AMBIGUOUS") { + const lowered = `${name} ${description}`.toLowerCase(); + for (const [lang, fileNode] of filesByLang) { + if (lowered.includes(lang) || confidence === "INFERRED") { + result.edges.push({ + source: skillId, + target: fileNode.id, + relation: "relevant_to", + confidence, + confidenceScore: score, + sourceFile: toPosixPath(skillMdPath), + sourceLocation: null, + lastVerified: now, + metadata: { miner: "plugin-miner", language: lang }, + }); + break; + } + } + } + } + } + + const agentsDir = join(entry.installPath, "agents"); + if (existsSync(agentsDir)) { + let agentFiles: string[] = []; + try { agentFiles = readdirSync(agentsDir); } catch { agentFiles = []; } + + for (const agentFile of agentFiles) { + if (!agentFile.endsWith(".md")) continue; + const agentPath = join(agentsDir, agentFile); + try { if (!statSync(agentPath).isFile()) continue; } catch { continue; } + + let content: string; + try { content = readFileSync(agentPath, "utf-8"); } catch { + result.anomalies.push(agentPath); + continue; + } + const fm = parseFrontmatter(content); + const name = fm.name || basename(agentFile, ".md"); + const description = fm.description || ""; + const { confidence, score } = scoreRelevance(name, description, stackTokens); + const agentId = `agent:${pluginName}/${name}`; + + result.nodes.push({ + id: agentId, + label: name, + kind: "concept", + sourceFile: toPosixPath(agentPath), + sourceLocation: null, + confidence, + confidenceScore: score, + lastVerified: now, + queryCount: 0, + metadata: { + miner: "plugin-miner", + subkind: "agent", + description, + sourcePlugin: pluginName, + marketplace, + }, + }); + + result.edges.push({ + source: agentId, + target: pluginId, + relation: "provided_by", + confidence: "EXTRACTED", + confidenceScore: 1.0, + sourceFile: toPosixPath(agentPath), + sourceLocation: null, + lastVerified: now, + metadata: { miner: "plugin-miner" }, + }); + } + } + } + + return result; +} diff --git a/tests/config-miner.test.ts b/tests/config-miner.test.ts new file mode 100644 index 0000000..31c14d5 --- /dev/null +++ b/tests/config-miner.test.ts @@ -0,0 +1,68 @@ +import { describe, it, expect } from "vitest"; +import { resolve } from "node:path"; +import { mineConfig } from "../src/miners/config-miner.js"; + +const FIXTURE_DIR = resolve(__dirname, "fixtures/settings"); +const GLOBAL = resolve(FIXTURE_DIR, "settings.json"); +const LOCAL = resolve(FIXTURE_DIR, "settings.local.json"); + +describe("mineConfig", () => { + it("returns empty when both files are missing", () => { + const result = mineConfig("/nope/settings.json", "/nope/local.json"); + expect(result.nodes).toHaveLength(0); + }); + + it("returns empty when ENGRAM_SKIP_ECOSYSTEM=1", () => { + process.env.ENGRAM_SKIP_ECOSYSTEM = "1"; + try { + const result = mineConfig(GLOBAL, LOCAL); + expect(result.nodes).toHaveLength(0); + } finally { + delete process.env.ENGRAM_SKIP_ECOSYSTEM; + } + }); + + it("indexes hooks from global settings", () => { + const result = mineConfig(GLOBAL, undefined); + const hooks = result.nodes.filter((n) => n.metadata.subkind === "hook"); + expect(hooks.length).toBeGreaterThanOrEqual(2); + const labels = hooks.map((h) => h.label); + expect(labels).toContain("SessionStart:startup"); + expect(labels).toContain("PreToolUse:*"); + }); + + it("indexes MCP servers from global settings only", () => { + const result = mineConfig(GLOBAL, LOCAL); + const mcps = result.nodes.filter((n) => n.metadata.subkind === "mcp_server"); + expect(mcps).toHaveLength(2); + const names = mcps.map((m) => m.label); + expect(names).toContain("context7"); + expect(names).toContain("playwright"); + }); + + it("merges hooks from global and local settings", () => { + const result = mineConfig(GLOBAL, LOCAL); + const hooks = result.nodes.filter((n) => n.metadata.subkind === "hook"); + const labels = hooks.map((h) => h.label); + expect(labels).toContain("UserPromptSubmit:*"); + }); + + it("all hook and mcp nodes have confidence 1.0", () => { + const result = mineConfig(GLOBAL, LOCAL); + for (const n of result.nodes) { + expect(n.confidence).toBe("EXTRACTED"); + expect(n.confidenceScore).toBe(1.0); + } + }); + + it("stores hook command in metadata", () => { + const result = mineConfig(GLOBAL, undefined); + const sessionStart = result.nodes.find((n) => n.label === "SessionStart:startup"); + expect(sessionStart?.metadata.command).toBe("engram intercept"); + }); + + it("handles malformed JSON silently", () => { + const result = mineConfig(FIXTURE_DIR, undefined); + expect(result.nodes).toHaveLength(0); + }); +}); diff --git a/tests/core.test.ts b/tests/core.test.ts index e44b6a0..37b9684 100644 --- a/tests/core.test.ts +++ b/tests/core.test.ts @@ -238,3 +238,15 @@ describe("Core — init lockfile guard (v0.2)", () => { expect(existsSync(join(tmpDir, ".engram", "init.lock"))).toBe(false); }); }); + +import { mineConfig } from "../src/miners/config-miner.js"; +import { minePlugins } from "../src/miners/plugin-miner.js"; + +describe("ecosystem miners integration", () => { + it("plugin-miner and config-miner are invokable with no-op inputs", () => { + const pluginResult = minePlugins("/nonexistent", []); + const configResult = mineConfig(undefined, undefined); + expect(pluginResult.nodes).toHaveLength(0); + expect(configResult.nodes).toHaveLength(0); + }); +}); diff --git a/tests/fixtures/claude-dir/plugins/installed_plugins.json b/tests/fixtures/claude-dir/plugins/installed_plugins.json new file mode 100644 index 0000000..12609b7 --- /dev/null +++ b/tests/fixtures/claude-dir/plugins/installed_plugins.json @@ -0,0 +1,14 @@ +{ + "plugins": { + "sample-plugin@mp": [ + { + "scope": "user", + "installPath": "FIXTURE_ABS_PATH", + "version": "1.2.3", + "installedAt": "2026-04-01T00:00:00Z", + "lastUpdated": "2026-04-10T00:00:00Z", + "gitCommitSha": "abc123" + } + ] + } +} diff --git a/tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/agents/reviewer.md b/tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/agents/reviewer.md new file mode 100644 index 0000000..7f89509 --- /dev/null +++ b/tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/agents/reviewer.md @@ -0,0 +1,6 @@ +--- +name: reviewer +description: General-purpose code review agent +--- + +Review code. diff --git a/tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/skills/python-review/SKILL.md b/tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/skills/python-review/SKILL.md new file mode 100644 index 0000000..d621038 --- /dev/null +++ b/tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/skills/python-review/SKILL.md @@ -0,0 +1,6 @@ +--- +name: python-review +description: Python code review with PEP 8 and type-hint checks +--- + +Review Python code. diff --git a/tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/skills/tdd/SKILL.md b/tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/skills/tdd/SKILL.md new file mode 100644 index 0000000..48f383e --- /dev/null +++ b/tests/fixtures/claude-dir/plugins/store/plugins/sample-plugin@mp/skills/tdd/SKILL.md @@ -0,0 +1,6 @@ +--- +name: tdd +description: Test-driven development workflow for any project +--- + +Write tests first. diff --git a/tests/fixtures/settings/settings.json b/tests/fixtures/settings/settings.json new file mode 100644 index 0000000..9ac1996 --- /dev/null +++ b/tests/fixtures/settings/settings.json @@ -0,0 +1,30 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup", + "hooks": [ + { "type": "command", "command": "engram intercept" } + ] + } + ], + "PreToolUse": [ + { + "matcher": "*", + "hooks": [ + { "type": "command", "command": "engram intercept" } + ] + } + ] + }, + "mcpServers": { + "context7": { + "command": "npx", + "args": ["-y", "@context7/mcp"] + }, + "playwright": { + "command": "npx", + "args": ["-y", "@playwright/mcp"] + } + } +} diff --git a/tests/fixtures/settings/settings.local.json b/tests/fixtures/settings/settings.local.json new file mode 100644 index 0000000..c578732 --- /dev/null +++ b/tests/fixtures/settings/settings.local.json @@ -0,0 +1,12 @@ +{ + "hooks": { + "UserPromptSubmit": [ + { + "matcher": "*", + "hooks": [ + { "type": "command", "command": "local-hook.sh" } + ] + } + ] + } +} diff --git a/tests/graph/stack-detect.test.ts b/tests/graph/stack-detect.test.ts new file mode 100644 index 0000000..87e1325 --- /dev/null +++ b/tests/graph/stack-detect.test.ts @@ -0,0 +1,85 @@ +import { describe, it, expect } from "vitest"; +import { detectStack } from "../../src/graph/stack-detect.js"; +import type { GraphNode } from "../../src/graph/schema.js"; + +function makeFileNode(sourceFile: string, label = sourceFile): GraphNode { + return { + id: `file:${sourceFile}`, + label, + kind: "file", + sourceFile, + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: 0, + queryCount: 0, + metadata: {}, + }; +} + +function makeClassNode(label: string): GraphNode { + return { + id: `class:${label}`, + label, + kind: "class", + sourceFile: "src/x.py", + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: 0, + queryCount: 0, + metadata: {}, + }; +} + +describe("detectStack", () => { + it("returns empty set for empty input", () => { + expect(detectStack([])).toEqual(new Set()); + }); + + it("detects python from .py files", () => { + const nodes = [makeFileNode("src/main.py")]; + expect(detectStack(nodes).has("python")).toBe(true); + }); + + it("detects typescript from .ts and .tsx files", () => { + const nodes = [makeFileNode("src/a.ts"), makeFileNode("src/b.tsx")]; + const stack = detectStack(nodes); + expect(stack.has("typescript")).toBe(true); + }); + + it("detects fastapi framework from class labels", () => { + const nodes = [makeFileNode("src/main.py"), makeClassNode("FastAPIRouter")]; + const stack = detectStack(nodes); + expect(stack.has("python")).toBe(true); + expect(stack.has("fastapi")).toBe(true); + }); + + it("detects mixed stack", () => { + const nodes = [ + makeFileNode("backend/main.py"), + makeFileNode("frontend/app.ts"), + ]; + const stack = detectStack(nodes); + expect(stack.has("python")).toBe(true); + expect(stack.has("typescript")).toBe(true); + }); + + it("ignores non-file non-class nodes for extension detection", () => { + const nodes: GraphNode[] = [ + { + id: "concept:foo", + label: "foo.py", + kind: "concept", + sourceFile: "", + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: 0, + queryCount: 0, + metadata: {}, + }, + ]; + expect(detectStack(nodes).has("python")).toBe(false); + }); +}); diff --git a/tests/plugin-miner-scoring.test.ts b/tests/plugin-miner-scoring.test.ts new file mode 100644 index 0000000..6478d90 --- /dev/null +++ b/tests/plugin-miner-scoring.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect } from "vitest"; +import { scoreRelevance } from "../src/miners/plugin-miner.js"; + +describe("scoreRelevance", () => { + it("returns INFERRED 0.6 when stack is empty", () => { + const result = scoreRelevance("python-tdd", "Python TDD workflow", new Set()); + expect(result.confidence).toBe("INFERRED"); + expect(result.score).toBe(0.6); + }); + + it("returns EXTRACTED 1.0 when skill name matches stack language", () => { + const stack = new Set(["python"]); + const result = scoreRelevance("python-tdd", "Python TDD workflow", stack); + expect(result.confidence).toBe("EXTRACTED"); + expect(result.score).toBe(1.0); + }); + + it("returns EXTRACTED 1.0 when skill matches stack framework", () => { + const stack = new Set(["python", "fastapi"]); + const result = scoreRelevance("fastapi-patterns", "FastAPI patterns", stack); + expect(result.confidence).toBe("EXTRACTED"); + }); + + it("returns AMBIGUOUS 0.2 when skill mentions non-matching language", () => { + const stack = new Set(["python"]); + const result = scoreRelevance("kotlin-review", "Kotlin code review", stack); + expect(result.confidence).toBe("AMBIGUOUS"); + expect(result.score).toBe(0.2); + }); + + it("returns INFERRED 0.6 for universal keywords when no language mention", () => { + const stack = new Set(["python"]); + const result = scoreRelevance("security-review", "Security audit", stack); + expect(result.confidence).toBe("INFERRED"); + expect(result.score).toBe(0.6); + }); + + it("returns AMBIGUOUS 0.2 when nothing matches", () => { + const stack = new Set(["python"]); + const result = scoreRelevance("random-thing", "unrelated content", stack); + expect(result.confidence).toBe("AMBIGUOUS"); + expect(result.score).toBe(0.2); + }); +}); diff --git a/tests/plugin-miner.test.ts b/tests/plugin-miner.test.ts new file mode 100644 index 0000000..1feddbf --- /dev/null +++ b/tests/plugin-miner.test.ts @@ -0,0 +1,133 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { mkdirSync, writeFileSync, readFileSync, rmSync, existsSync, readdirSync, statSync, copyFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { minePlugins } from "../src/miners/plugin-miner.js"; +import type { GraphNode } from "../src/graph/schema.js"; + +const FIXTURE_SRC = resolve(__dirname, "fixtures/claude-dir"); + +function copyDir(src: string, dst: string): void { + if (!existsSync(dst)) mkdirSync(dst, { recursive: true }); + for (const entry of readdirSync(src)) { + const s = join(src, entry); + const d = join(dst, entry); + if (statSync(s).isDirectory()) copyDir(s, d); + else copyFileSync(s, d); + } +} + +function setupFixture(): string { + const tmp = join(tmpdir(), `engram-plugin-miner-${Date.now()}-${Math.random()}`); + mkdirSync(tmp, { recursive: true }); + copyDir(FIXTURE_SRC, tmp); + const manifestPath = join(tmp, "plugins", "installed_plugins.json"); + const manifest = readFileSync(manifestPath, "utf-8"); + const pluginAbs = join(tmp, "plugins", "store", "plugins", "sample-plugin@mp"); + const escaped = pluginAbs.replace(/\\/g, "\\\\"); + writeFileSync(manifestPath, manifest.replace("FIXTURE_ABS_PATH", escaped)); + return tmp; +} + +function pyFileNode(path: string): GraphNode { + return { + id: `file:${path}`, + label: path, + kind: "file", + sourceFile: path, + sourceLocation: null, + confidence: "EXTRACTED", + confidenceScore: 1.0, + lastVerified: 0, + queryCount: 0, + metadata: {}, + }; +} + +describe("minePlugins", () => { + let claudeDir: string; + + beforeAll(() => { + claudeDir = setupFixture(); + }); + + afterAll(() => { + rmSync(claudeDir, { recursive: true, force: true }); + }); + + it("returns empty when claudeDir is missing", () => { + const result = minePlugins("/does/not/exist", []); + expect(result.nodes).toHaveLength(0); + expect(result.edges).toHaveLength(0); + expect(result.pluginCount).toBe(0); + }); + + it("returns empty when ENGRAM_SKIP_ECOSYSTEM=1", () => { + process.env.ENGRAM_SKIP_ECOSYSTEM = "1"; + try { + const result = minePlugins(claudeDir, [pyFileNode("main.py")]); + expect(result.nodes).toHaveLength(0); + expect(result.pluginCount).toBe(0); + } finally { + delete process.env.ENGRAM_SKIP_ECOSYSTEM; + } + }); + + it("indexes plugin, its 2 skills, and 1 agent", () => { + const result = minePlugins(claudeDir, [pyFileNode("main.py")]); + expect(result.pluginCount).toBe(1); + const pluginNodes = result.nodes.filter((n) => n.metadata.subkind === "plugin"); + const skillNodes = result.nodes.filter((n) => n.metadata.subkind === "skill"); + const agentNodes = result.nodes.filter((n) => n.metadata.subkind === "agent"); + expect(pluginNodes).toHaveLength(1); + expect(skillNodes).toHaveLength(2); + expect(agentNodes).toHaveLength(1); + }); + + it("creates provided_by edges from skill/agent to plugin", () => { + const result = minePlugins(claudeDir, [pyFileNode("main.py")]); + const providedBy = result.edges.filter((e) => e.relation === "provided_by"); + expect(providedBy).toHaveLength(3); + for (const e of providedBy) { + expect(e.target).toBe("plugin:sample-plugin"); + } + }); + + it("scores python-review as EXTRACTED when project has python files", () => { + const result = minePlugins(claudeDir, [pyFileNode("main.py")]); + const pyReview = result.nodes.find((n) => n.label === "python-review"); + expect(pyReview?.confidence).toBe("EXTRACTED"); + }); + + it("creates relevant_to edges only for EXTRACTED or INFERRED skills", () => { + const result = minePlugins(claudeDir, [pyFileNode("main.py")]); + const relevantTo = result.edges.filter((e) => e.relation === "relevant_to"); + for (const e of relevantTo) { + const src = result.nodes.find((n) => n.id === e.source); + expect(src?.confidence).not.toBe("AMBIGUOUS"); + } + }); + + it("handles plugin directory without skills/ or agents/ gracefully", () => { + const emptyPluginDir = join(claudeDir, "plugins", "store", "plugins", "empty-plugin@mp"); + mkdirSync(emptyPluginDir, { recursive: true }); + const manifestPath = join(claudeDir, "plugins", "installed_plugins.json"); + const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")); + manifest.plugins["empty-plugin@mp"] = [ + { + scope: "user", + installPath: emptyPluginDir, + version: "0.1.0", + installedAt: "2026-04-01T00:00:00Z", + lastUpdated: "2026-04-01T00:00:00Z", + gitCommitSha: "def456", + }, + ]; + writeFileSync(manifestPath, JSON.stringify(manifest)); + + const result = minePlugins(claudeDir, [pyFileNode("main.py")]); + expect(result.pluginCount).toBe(2); + const emptyPluginNode = result.nodes.find((n) => n.id === "plugin:empty-plugin"); + expect(emptyPluginNode).toBeDefined(); + }); +});