From f314082e1d05e553f01ca7cc8da8b1d45f1450cc Mon Sep 17 00:00:00 2001 From: Dean Sharon Date: Fri, 20 Mar 2026 09:52:39 +0200 Subject: [PATCH] fix(memory): replace --resume with transcript-based background updater The background working memory updater has been broken for 17+ days because `claude --resume` loads the full conversation, causing the model to respond conversationally instead of calling Write. Replace with transcript reading + fresh `claude -p` invocation, and remove the redundant PROJECT-PATTERNS.md system (superseded by knowledge-persistence skill's decisions.md + pitfalls.md). --- CLAUDE.md | 3 +- README.md | 1 - docs/reference/file-organization.md | 2 +- scripts/hooks/background-memory-update | 199 ++++++++++++++----------- scripts/hooks/session-start-memory | 18 +-- shared/skills/docs-framework/SKILL.md | 1 - src/cli/utils/post-install.ts | 1 - tests/memory.test.ts | 9 +- 8 files changed, 119 insertions(+), 115 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index db28b05..b2a1bb0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,7 +38,7 @@ Commands with Teams Variant ship as `{name}.md` (parallel subagents) and `{name} **Build-time asset distribution**: Skills and agents are stored once in `shared/skills/` and `shared/agents/`, then copied to each plugin at build time based on `plugin.json` manifests. This eliminates duplication in git. -**Working Memory**: Three shell-script hooks (`scripts/hooks/`) provide automatic session continuity. Toggleable via `devflow memory --enable/--disable/--status` or `devflow init --memory/--no-memory`. Stop hook → spawns a background `claude -p --resume` process that asynchronously updates `.memory/WORKING-MEMORY.md` with structured sections (`## Now`, `## Progress`, `## Decisions`, `## Modified Files`, `## Context`, `## Session Log`; throttled: skips if updated <2min ago; concurrent sessions serialize via mkdir-based lock; restricted to Read+Write on two specific files + read-only git commands via `--tools`/`--allowedTools`). SessionStart hook → injects previous memory + git state as `additionalContext` on `/clear`, startup, or compact (warns if >1h stale; injects pre-compact memory snapshot when compaction happened mid-session). PreCompact hook → saves git state + WORKING-MEMORY.md snapshot + bootstraps minimal WORKING-MEMORY.md if none exists. Zero-ceremony context preservation. +**Working Memory**: Three shell-script hooks (`scripts/hooks/`) provide automatic session continuity. Toggleable via `devflow memory --enable/--disable/--status` or `devflow init --memory/--no-memory`. Stop hook → reads last turn from session transcript (`~/.claude/projects/{encoded-cwd}/{session_id}.jsonl`), spawns background `claude -p --model haiku` to update `.memory/WORKING-MEMORY.md` with structured sections (`## Now`, `## Progress`, `## Decisions`, `## Modified Files`, `## Context`, `## Session Log`; throttled: skips if triggered <2min ago; concurrent sessions serialize via mkdir-based lock). SessionStart hook → injects previous memory + git state as `additionalContext` on `/clear`, startup, or compact (warns if >1h stale; injects pre-compact memory snapshot when compaction happened mid-session). PreCompact hook → saves git state + WORKING-MEMORY.md snapshot + bootstraps minimal WORKING-MEMORY.md if none exists. Zero-ceremony context preservation. ## Project Structure @@ -93,7 +93,6 @@ Working memory files live in a dedicated `.memory/` directory: ``` .memory/ ├── WORKING-MEMORY.md # Auto-maintained by Stop hook (overwritten each session) -├── PROJECT-PATTERNS.md # Accumulated patterns (merged, not overwritten) ├── backup.json # Pre-compact git state snapshot └── knowledge/ ├── decisions.md # Architectural decisions (ADR-NNN, append-only) diff --git a/README.md b/README.md index 04c11b6..8937852 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,6 @@ DevFlow creates project documentation in `.docs/` and working memory in `.memory .memory/ ├── WORKING-MEMORY.md # Auto-maintained by Stop hook -├── PROJECT-PATTERNS.md # Accumulated patterns across sessions ├── backup.json # Pre-compact git state snapshot └── knowledge/ ├── decisions.md # Architectural decisions (ADR-NNN, append-only) diff --git a/docs/reference/file-organization.md b/docs/reference/file-organization.md index a802d6e..4064bc0 100644 --- a/docs/reference/file-organization.md +++ b/docs/reference/file-organization.md @@ -145,7 +145,7 @@ Three hooks in `scripts/hooks/` provide automatic session continuity. Toggleable | `session-start-memory` | SessionStart | reads WORKING-MEMORY.md | Injects previous memory + git state as `additionalContext`. Warns if >1h stale. Injects pre-compact snapshot when compaction occurred mid-session. | | `pre-compact-memory` | PreCompact | `.memory/backup.json` | Saves git state + WORKING-MEMORY.md snapshot. Bootstraps minimal WORKING-MEMORY.md if none exists. | -**Flow**: Claude responds → Stop hook checks mtime (skips if <2min fresh) → blocks with instruction → Claude writes WORKING-MEMORY.md silently → `stop_hook_active=true` → allows stop. On `/clear` or new session → SessionStart injects memory as `additionalContext` (system context, not user-visible) with staleness warning if >1h old. +**Flow**: Session ends → Stop hook checks throttle (skips if <2min fresh) → spawns background updater → background updater reads session transcript + git state → fresh `claude -p --model haiku` writes WORKING-MEMORY.md. On `/clear` or new session → SessionStart injects memory as `additionalContext` (system context, not user-visible) with staleness warning if >1h old. Hooks auto-create `.memory/` on first run — no manual setup needed per project. diff --git a/scripts/hooks/background-memory-update b/scripts/hooks/background-memory-update index 49fa7ab..2ba5749 100755 --- a/scripts/hooks/background-memory-update +++ b/scripts/hooks/background-memory-update @@ -2,8 +2,9 @@ # Background Working Memory Updater # Called by stop-update-memory as a detached background process. -# Resumes the parent session headlessly to update .memory/WORKING-MEMORY.md. -# On failure: logs error, does nothing (no fallback). +# Reads the last turn from the session transcript, then uses a fresh `claude -p` +# invocation to update .memory/WORKING-MEMORY.md. +# On failure: logs error, does nothing (stale memory is better than fake data). set -e @@ -29,7 +30,7 @@ rotate_log() { # --- Stale Lock Recovery --- -# Portable mtime in epoch seconds (same pattern as stop-update-memory:35-39) +# Portable mtime in epoch seconds get_mtime() { if stat --version &>/dev/null 2>&1; then stat -c %Y "$1" @@ -72,11 +73,65 @@ cleanup() { } trap cleanup EXIT +# --- Transcript Extraction --- + +extract_last_turn() { + # Compute transcript path: Claude Code stores transcripts at + # ~/.claude/projects/{cwd-with-slashes-replaced-by-hyphens}/{session_id}.jsonl + local encoded_cwd + encoded_cwd=$(echo "$CWD" | sed 's|^/||' | tr '/' '-') + local transcript="$HOME/.claude/projects/-${encoded_cwd}/${SESSION_ID}.jsonl" + + if [ ! -f "$transcript" ]; then + log "Transcript not found at $transcript" + return 1 + fi + + # Extract last user and assistant text from JSONL + # Each line is a JSON object with "type" field + local last_user last_assistant + + last_user=$(grep '"type":"user"' "$transcript" 2>/dev/null \ + | tail -3 \ + | jq -r ' + if .message.content then + [.message.content[] | select(.type == "text") | .text] | join("\n") + else "" + end + ' 2>/dev/null \ + | tail -1) + + last_assistant=$(grep '"type":"assistant"' "$transcript" 2>/dev/null \ + | tail -3 \ + | jq -r ' + if .message.content then + [.message.content[] | select(.type == "text") | .text] | join("\n") + else "" + end + ' 2>/dev/null \ + | tail -1) + + # Truncate to ~4000 chars total to keep token cost low + if [ ${#last_user} -gt 2000 ]; then + last_user="${last_user:0:2000}... [truncated]" + fi + if [ ${#last_assistant} -gt 2000 ]; then + last_assistant="${last_assistant:0:2000}... [truncated]" + fi + + if [ -z "$last_user" ] && [ -z "$last_assistant" ]; then + log "No text content found in transcript" + return 1 + fi + + LAST_USER_TEXT="$last_user" + LAST_ASSISTANT_TEXT="$last_assistant" + return 0 +} + # --- Main --- -# Wait for parent session to flush transcript. -# 3s provides ~6-10x margin over typical flush times. -# If --resume shows stale transcripts, bump to 5s. +# Wait for parent session to flush transcript sleep 3 log "Starting update for session $SESSION_ID" @@ -87,7 +142,6 @@ break_stale_lock # Acquire lock (other sessions may be updating concurrently) if ! acquire_lock; then log "Lock timeout after 90s — skipping update for session $SESSION_ID" - # Don't clean up lock we don't own trap - EXIT exit 0 fi @@ -102,97 +156,72 @@ if [ -f "$MEMORY_FILE" ]; then PRE_UPDATE_MTIME=$(get_mtime "$MEMORY_FILE") fi -# Build instruction -if [ -n "$EXISTING_MEMORY" ]; then - PATTERNS_INSTRUCTION="" -PATTERNS_FILE="$CWD/.memory/PROJECT-PATTERNS.md" -EXISTING_PATTERNS="" -if [ -f "$PATTERNS_FILE" ]; then - EXISTING_PATTERNS=$(cat "$PATTERNS_FILE") - PATTERNS_INSTRUCTION=" - -Also update $PATTERNS_FILE by APPENDING any new recurring patterns discovered during this session. Do NOT overwrite existing entries — only add new ones. Skip if no new patterns were observed. Format each entry as: - **Pattern name**: Brief description (discovered: YYYY-MM-DD). Keep patterns.md under 40 entries. When approaching the limit, consolidate related patterns into broader entries rather than adding duplicates. - -Existing patterns: -$EXISTING_PATTERNS" -else - PATTERNS_INSTRUCTION=" - -If recurring patterns were observed during this session (coding conventions, architectural decisions, team preferences, tooling quirks), create $PATTERNS_FILE with entries formatted as: - **Pattern name**: Brief description (discovered: YYYY-MM-DD). Only create this file if genuine patterns were observed — do not fabricate entries." +# Gather git state (always available, used as fallback too) +GIT_STATE="" +if cd "$CWD" 2>/dev/null && git rev-parse --git-dir >/dev/null 2>&1; then + GIT_STATUS=$(git status --short 2>/dev/null | head -20) + GIT_LOG=$(git log --oneline -5 2>/dev/null) + GIT_DIFF=$(git diff --stat HEAD 2>/dev/null | tail -10) + GIT_STATE="Branch: $(git branch --show-current 2>/dev/null || echo 'unknown') +Recent commits: +${GIT_LOG} +Changed files: +${GIT_STATUS} +Diff summary: +${GIT_DIFF}" fi -INSTRUCTION="First, Read the file $MEMORY_FILE to satisfy Claude Code's read-before-write requirement. Then update it with working memory from this session. The file already has content — possibly from a concurrent session that just wrote it moments ago. Merge this session's context with the existing content to produce a single unified working memory snapshot. Both this session and the existing content represent fresh, concurrent work — integrate both fully. Working memory captures what's active now, not a changelog. Deduplicate overlapping information. Keep under 120 lines total. Use the same structure: ## Now, ## Progress, ## Decisions, ## Modified Files, ## Context, ## Session Log. - -## Progress tracks Done (completed items), Remaining (next steps), and Blockers (if any). Keep each sub-list to 1-3 items. This section reflects current work state, not historical logs. - -## Decisions entries must include date and status. Format: - **[Decision]** — [rationale] (YYYY-MM-DD) [ACTIVE|SUPERSEDED]. Mark superseded decisions rather than deleting them.${PATTERNS_INSTRUCTION} - -Existing content: -$EXISTING_MEMORY" +# Extract last turn from transcript (or fall back to git-only) +LAST_USER_TEXT="" +LAST_ASSISTANT_TEXT="" +EXCHANGE_SECTION="" + +if extract_last_turn; then + log "--- Extracted user text (${#LAST_USER_TEXT} chars) ---" + log "$LAST_USER_TEXT" + log "--- Extracted assistant text (${#LAST_ASSISTANT_TEXT} chars) ---" + log "$LAST_ASSISTANT_TEXT" + log "--- End transcript extraction ---" + EXCHANGE_SECTION="Last exchange: +User: ${LAST_USER_TEXT} +Assistant: ${LAST_ASSISTANT_TEXT}" else - PATTERNS_INSTRUCTION="" - PATTERNS_FILE="$CWD/.memory/PROJECT-PATTERNS.md" - if [ -f "$PATTERNS_FILE" ]; then - EXISTING_PATTERNS=$(cat "$PATTERNS_FILE") - PATTERNS_INSTRUCTION=" - -Also update $PATTERNS_FILE by APPENDING any new recurring patterns discovered during this session. Do NOT overwrite existing entries — only add new ones. Skip if no new patterns were observed. Format each entry as: - **Pattern name**: Brief description (discovered: YYYY-MM-DD). Keep patterns.md under 40 entries. When approaching the limit, consolidate related patterns into broader entries rather than adding duplicates. - -Existing patterns: -$EXISTING_PATTERNS" - else - PATTERNS_INSTRUCTION=" - -If recurring patterns were observed during this session (coding conventions, architectural decisions, team preferences, tooling quirks), create $PATTERNS_FILE with entries formatted as: - **Pattern name**: Brief description (discovered: YYYY-MM-DD). Only create this file if genuine patterns were observed — do not fabricate entries." - fi - - INSTRUCTION="First, Read the file $MEMORY_FILE if it exists (to satisfy Claude Code's read-before-write requirement). Then create it with working memory from this session. Keep under 120 lines. Use this structure: - -# Working Memory - -## Now - - -## Progress - + log "Falling back to git-state-only context" + EXCHANGE_SECTION="(Session transcript not available — using git state only)" +fi -## Decisions - +# Build prompt for fresh claude -p invocation +PROMPT="You are a working memory updater. Your ONLY job is to update the file at ${MEMORY_FILE} using the Write tool. Do it immediately — do not ask questions or explain. -## Modified Files - +Current working memory: +${EXISTING_MEMORY:-"(no existing content)"} -## Context - +${EXCHANGE_SECTION} -## Session Log +Git state: +${GIT_STATE:-"(not a git repo)"} -### Today - +Instructions: +- Use the Write tool to update ${MEMORY_FILE} immediately +- Keep under 120 lines +- Use sections: ## Now, ## Progress, ## Decisions, ## Modified Files, ## Context, ## Session Log +- Integrate new information with existing content +- Deduplicate overlapping information +- ## Progress tracks Done (completed), Remaining (next steps), Blockers (if any) +- ## Decisions entries: format as - **[Decision]** — [rationale] (YYYY-MM-DD) [ACTIVE|SUPERSEDED]" -### This Week -${PATTERNS_INSTRUCTION}" -fi +log "--- Full prompt being passed to claude -p ---" +log "$PROMPT" +log "--- End prompt ---" -# Resume session headlessly to perform the update -TIMEOUT=120 # Normal runtime 30-60s; 2x margin +# Run fresh claude -p (no --resume, no conversation confusion) +TIMEOUT=120 -DEVFLOW_BG_UPDATER=1 env -u CLAUDECODE "$CLAUDE_BIN" -p \ - --resume "$SESSION_ID" \ +DEVFLOW_BG_UPDATER=1 "$CLAUDE_BIN" -p \ --model haiku \ - --tools "Read,Write,Bash" \ - --allowedTools \ - "Read($CWD/.memory/WORKING-MEMORY.md)" \ - "Read($CWD/.memory/PROJECT-PATTERNS.md)" \ - "Write($CWD/.memory/WORKING-MEMORY.md)" \ - "Write($CWD/.memory/PROJECT-PATTERNS.md)" \ - "Bash(git status:*)" \ - "Bash(git log:*)" \ - "Bash(git diff:*)" \ - "Bash(git branch:*)" \ - --no-session-persistence \ + --dangerously-skip-permissions \ --output-format text \ - "$INSTRUCTION" \ + "$PROMPT" \ >> "$LOG_FILE" 2>&1 & CLAUDE_PID=$! diff --git a/scripts/hooks/session-start-memory b/scripts/hooks/session-start-memory index f70613e..62d294a 100644 --- a/scripts/hooks/session-start-memory +++ b/scripts/hooks/session-start-memory @@ -2,7 +2,7 @@ # SessionStart Hook # Injects working memory AND ambient skill content as additionalContext. -# Memory: restores .memory/WORKING-MEMORY.md + patterns + git state + compact recovery. +# Memory: restores .memory/WORKING-MEMORY.md + git state + compact recovery. # Ambient: injects ambient-router SKILL.md so Claude has it in context (no Read call needed). # Either section can fire independently — ambient works even without memory files. @@ -27,13 +27,6 @@ MEMORY_FILE="$CWD/.memory/WORKING-MEMORY.md" if [ -f "$MEMORY_FILE" ]; then MEMORY_CONTENT=$(cat "$MEMORY_FILE") - # Read accumulated patterns if they exist - PATTERNS_FILE="$CWD/.memory/PROJECT-PATTERNS.md" - PATTERNS_CONTENT="" - if [ -f "$PATTERNS_FILE" ]; then - PATTERNS_CONTENT=$(cat "$PATTERNS_FILE") - fi - # Compute staleness warning if stat --version &>/dev/null 2>&1; then FILE_MTIME=$(stat -c %Y "$MEMORY_FILE") @@ -91,15 +84,6 @@ $BACKUP_MEMORY ${MEMORY_CONTENT}" - # Insert accumulated patterns between working memory and git state - if [ -n "$PATTERNS_CONTENT" ]; then - CONTEXT="${CONTEXT} - ---- PROJECT PATTERNS (accumulated) --- - -${PATTERNS_CONTENT}" - fi - CONTEXT="${CONTEXT} --- CURRENT GIT STATE --- diff --git a/shared/skills/docs-framework/SKILL.md b/shared/skills/docs-framework/SKILL.md index 573b4b1..c8af66d 100644 --- a/shared/skills/docs-framework/SKILL.md +++ b/shared/skills/docs-framework/SKILL.md @@ -38,7 +38,6 @@ All generated documentation lives under `.docs/` in the project root: .memory/ ├── WORKING-MEMORY.md # Auto-maintained by Stop hook (overwritten) -├── PROJECT-PATTERNS.md # Accumulated patterns (merged across sessions) ├── backup.json # Pre-compact git state snapshot └── knowledge/ ├── decisions.md # Architectural decisions (ADR-NNN format) diff --git a/src/cli/utils/post-install.ts b/src/cli/utils/post-install.ts index 6ed54d8..a5321e4 100644 --- a/src/cli/utils/post-install.ts +++ b/src/cli/utils/post-install.ts @@ -502,7 +502,6 @@ export async function migrateMemoryFiles(verbose: boolean, cwd?: string): Promis const migrations: Array<{ src: string; dest: string }> = [ { src: path.join(docsDir, 'WORKING-MEMORY.md'), dest: path.join(memoryDir, 'WORKING-MEMORY.md') }, - { src: path.join(docsDir, 'patterns.md'), dest: path.join(memoryDir, 'PROJECT-PATTERNS.md') }, { src: path.join(docsDir, 'working-memory-backup.json'), dest: path.join(memoryDir, 'backup.json') }, ]; diff --git a/tests/memory.test.ts b/tests/memory.test.ts index d332d94..5f9a7c8 100644 --- a/tests/memory.test.ts +++ b/tests/memory.test.ts @@ -298,29 +298,24 @@ describe('migrateMemoryFiles', () => { expect(count).toBe(0); }); - it('migrates all 3 files from .docs/ to .memory/', async () => { + it('migrates memory files from .docs/ to .memory/', async () => { const docsDir = path.join(tmpDir, '.docs'); await fs.mkdir(docsDir, { recursive: true }); await fs.writeFile(path.join(docsDir, 'WORKING-MEMORY.md'), '# Working Memory'); - await fs.writeFile(path.join(docsDir, 'patterns.md'), '# Patterns'); await fs.writeFile(path.join(docsDir, 'working-memory-backup.json'), '{}'); const count = await migrateMemoryFiles(false, tmpDir); - expect(count).toBe(3); + expect(count).toBe(2); // Verify destinations exist const wm = await fs.readFile(path.join(tmpDir, '.memory', 'WORKING-MEMORY.md'), 'utf-8'); expect(wm).toBe('# Working Memory'); - const patterns = await fs.readFile(path.join(tmpDir, '.memory', 'PROJECT-PATTERNS.md'), 'utf-8'); - expect(patterns).toBe('# Patterns'); - const backup = await fs.readFile(path.join(tmpDir, '.memory', 'backup.json'), 'utf-8'); expect(backup).toBe('{}'); // Verify sources removed await expect(fs.access(path.join(docsDir, 'WORKING-MEMORY.md'))).rejects.toThrow(); - await expect(fs.access(path.join(docsDir, 'patterns.md'))).rejects.toThrow(); await expect(fs.access(path.join(docsDir, 'working-memory-backup.json'))).rejects.toThrow(); });