feat: DumpMergeWriter + merge-aware compaction

.claude/skills/deploy/cli.mjs

@@ -365,15 +365,76 @@ function resources() {
 }
 
 function wipe() {
-  err('Wiping bitmap/docstore data on both PVCs (keeping CSVs)...');
+  err('Wiping bitmap/docstore/WAL data on both PVCs (keeping CSVs in load_stage)...');
+  err('  IMPORTANT: Pod must be scaled to 0 first, or server recreates dirs on boot.');
   for (const i of [0, 1]) {
-    const cmd = `rm -rf ${INDEX_PATH}/bitmaps ${INDEX_PATH}/docs ${INDEX_PATH}/bounds ${INDEX_PATH}/slot_arena.bin ${INDEX_PATH}/snapshot.meta && echo wiped-${i}`;
-    const { logs } = runEphemeralPod(`bitdex-wipe-${i}`, { command: cmd, pvcIndex: i, timeout: 60 });
+    // rm -rf everything EXCEPT load_stage CSVs.
+    // Must use rm -rf (not find -delete) because the server recreates empty dirs
+    // on boot, and find -delete would leave shard files inside recreated dirs.
+    // WAL cleanup is CRITICAL: stale WAL cursor in MetaStore = broken WAL reader.
+    const cmd = [
+      `rm -rf ${INDEX_PATH}/bitmaps`,    // ShardStore: alive, filter, sort + MetaStore cursors
+      `rm -rf ${INDEX_PATH}/docs`,        // DocStore V3 shard files
+      `rm -rf /data/wal`,                 // WAL files (ops_000001.wal etc.)
+      `mkdir -p /data/wal`,               // Recreate empty WAL dir for next boot
+      `rm -rf ${INDEX_PATH}/bounds`,      // Bound cache shards
+      `rm -f ${INDEX_PATH}/dumps.json`,   // Dump state — forces fresh dump on restart
+      `rm -f ${INDEX_PATH}/slot_arena.bin`,
+      `rm -f ${INDEX_PATH}/snapshot.meta`,
+      `echo wiped-${i}`,
+    ].join(' && ');
+    const { logs } = runEphemeralPod(`bitdex-wipe-${i}`, { command: cmd, pvcIndex: i, timeout: 120 });
     err(`  PVC ${i}: ${logs}`);
   }
   json({ wiped: true });
 }
 
+function fullReset() {
+  err('=== FULL RESET: nuke + PG cleanup (keeps CSVs) ===');
+
+  // Step 1: Scale to 0
+  err('Step 1: Scaling to 0...');
+  run(`kubectl scale sts ${STS} -n ${NS} --replicas=0 --context ${K8S_CONTEXT} 2>/dev/null`);
+  err('  Waiting for pods to terminate...');
+  run(`kubectl wait --for=delete pod/bitdex-0 -n ${NS} --timeout=120s --context ${K8S_CONTEXT} 2>/dev/null`, { throws: false });
+
+  // Step 2: Wipe PVC data (keeps CSVs)
+  err('Step 2: Wiping PVC data (keeping load_stage CSVs)...');
+  wipe();
+
+  // Step 3: PG cleanup — drop triggers, truncate ops, delete cursors
+  err('Step 3: PG cleanup...');
+  const pgCmds = [
+    // Drop V2 triggers (they'll be recreated by bitdex-sync on startup)
+    `DO $$ DECLARE r RECORD; BEGIN FOR r IN SELECT tgname, relname FROM pg_trigger t JOIN pg_class c ON t.tgrelid = c.oid WHERE tgname LIKE 'bitdex_%' LOOP EXECUTE format('DROP TRIGGER IF EXISTS %I ON %I', r.tgname, r.relname); END LOOP; END $$`,
+    // Truncate ops table
+    `TRUNCATE TABLE "BitdexOps"`,
+    // Delete all cursors
+    `DELETE FROM bitdex_cursors`,
+  ];
+  for (const cmd of pgCmds) {
+    const result = run(
+      `MSYS_NO_PATHCONV=1 kubectl exec -n ${PG_NS} ${PG_POD} --context ${K8S_CONTEXT} -- psql -U postgres -d civitai -c "${cmd.replace(/"/g, '\\"')}" 2>/dev/null`,
+      { throws: false }
+    );
+    err(`  PG: ${result || 'ok'}`);
+  }
+
+  // Step 4: Scale back to 1
+  err('Step 4: Scaling to 1...');
+  run(`kubectl scale sts ${STS} -n ${NS} --replicas=1 --context ${K8S_CONTEXT} 2>/dev/null`);
+  err('  Waiting for pod to be ready...');
+  run(`kubectl wait --for=condition=ready pod/bitdex-0 -n ${NS} --timeout=300s --context ${K8S_CONTEXT} 2>/dev/null`, { throws: false });
+
+  err('=== FULL RESET COMPLETE ===');
+  err('  PVC: wiped (bitmaps, docs, WAL, bounds, dumps.json) — rm -rf, not find -delete');
+  err('  PG: triggers dropped, ops truncated, cursors deleted');
+  err('  WAL: deleted + empty /data/wal/ recreated');
+  err('  CSVs: preserved in load_stage');
+  err('  Next: pod will boot fresh, sidecar will re-create triggers and start dump from CSVs');
+  json({ reset: true, steps: ['scale_down', 'wipe_pvc', 'pg_cleanup', 'scale_up'] });
+}
+
 function configRead() {
   const config = kubectlExec(`cat ${INDEX_PATH}/config.json`);
   try { json(JSON.parse(config)); } catch { json({ error: 'Could not read config', raw: config }); }
@@ -751,6 +812,7 @@ switch (command) {
   // Operations
   case 'resources': resources(); break;
   case 'wipe': wipe(); break;
+  case 'full-reset': fullReset(); break;
   case 'config-read': configRead(); break;
   case 'config-patch': configPatch(); break;
   case 'memory': memory(); break;
@@ -825,7 +887,7 @@ switch (command) {
         'Tunnels': ['tunnel pg [start|stop|status]', 'tunnel bitdex [start|stop|status]'],
         'Snapshots': ['snapshot-status <session_id>', 'snapshot-download <session_id> [--output <path>]'],
         'Metrics': ['metrics-now', 'metrics-trend [window]', 'metrics-query <promql>'],
-        'Data': ['wipe', 'cleanup <captures|load_stage|legacy|bounds>'],
+        'Data': ['wipe', 'full-reset', 'cleanup <captures|load_stage|legacy|bounds>'],
       },
     });
     process.exit(1);

.claude/skills/ralph/.gitignore

@@ -0,0 +1,13 @@
+# Project folders contain PRDs and progress files - not committed
+projects/
+
+# Legacy root-level files (if anyone runs without --prd)
+prd.json
+progress.txt
+
+# Daemon runtime files
+daemon/daemon.pid
+daemon/data/
+
+# Temp directories created by agent sessions
+tmpclaude-*-cwd

.claude/skills/ralph/SKILL.md

@@ -0,0 +1,120 @@
+---
+name: ralph
+description: Autonomous agent for tackling big projects. Create PRDs with user stories, then run them via the CLI. Sessions persist across restarts with pause/resume and real-time monitoring.
+---
+
+# Ralph - Autonomous Agent
+
+Ralph breaks big projects into user stories and executes them autonomously. The workflow:
+
+1. **Create a PRD** - Define user stories with acceptance criteria
+2. **Run it** - `ralph.mjs create --prd path/to/prd.json --start`
+3. **Monitor** - `ralph.mjs logs <session-id> --follow`
+
+## Creating a PRD
+
+Create a project folder and prd.json:
+```
+.claude/skills/ralph/projects/<project-name>/prd.json
+```
+
+### PRD Structure
+
+```json
+{
+  "description": "Brief description of the feature",
+  "branchName": "feature/my-feature",
+  "userStories": [
+    {
+      "id": "US001",
+      "title": "Short descriptive title",
+      "description": "As a [user], I want [feature] so that [benefit]",
+      "acceptanceCriteria": [
+        "Specific testable criterion",
+        "Typecheck passes"
+      ],
+      "priority": 1,
+      "passes": false
+    }
+  ]
+}
+```
+
+### Story Guidelines
+
+- **Priority 1**: Foundation - migrations, types, base components
+- **Priority 2-3**: Core functionality
+- **Priority 4+**: Secondary features, polish
+- Each story should touch 1-3 files, not 10-file refactors
+- Include "Typecheck passes" in acceptance criteria
+
+## CLI Commands
+
+The daemon starts automatically when you run any command.
+
+### Running Sessions
+
+```bash
+# Create and start a session
+ralph.mjs create --prd path/to/prd.json --start
+
+# List all sessions
+ralph.mjs list
+
+# Check session status
+ralph.mjs status <session-id>
+
+# Follow logs in real-time
+ralph.mjs logs <session-id> --follow
+```
+
+### Session Control
+
+```bash
+# Pause a session
+ralph.mjs pause <session-id> --reason "Waiting for API"
+
+# Resume with guidance
+ralph.mjs resume <session-id> --guidance "API is ready on port 3000"
+
+# Inject guidance into running session
+ralph.mjs inject <session-id> --message "Try using the helper in utils.ts"
+
+# Abort a session
+ralph.mjs abort <session-id>
+```
+
+### Orchestration (Multi-Level)
+
+For orchestrator PRDs that spawn child sessions:
+
+```bash
+# Spawn a child session
+ralph.mjs spawn <parent-id> --prd child/prd.json --start
+
+# List children of a session
+ralph.mjs children <session-id>
+
+# Wait for all children to complete
+ralph.mjs wait <session-id>
+
+# View session tree
+ralph.mjs tree <session-id>
+
+# Abort parent and all children
+ralph.mjs abort <session-id> --cascade
+```
+
+## PRD Types
+
+| Type | Use Case |
+|------|----------|
+| `code` (default) | Implement features, commit code |
+| `orchestrator` | Coordinate multiple sub-Ralphs |
+| `testing` | Browser automation testing |
+
+Set via `"type": "orchestrator"` in prd.json.
+
+## Full CLI Reference
+
+Run `ralph.mjs --help` for complete documentation.