(dev -> main) App updates and backwards transformer ci functions

.env.example

@@ -1,2 +1,10 @@
 WANDB_API_KEY=your_api_key
 WANDB_ENTITY=your_entity_name
+
+# LLM API keys for autointerp (set whichever you need based on model choice)
+# Models with "/" in name (e.g. google/gemini-3.1-pro-preview) use OpenRouter
+# Models starting with "claude-" use Anthropic directly
+# Models starting with "gpt-"/"o1-"/"o3-"/"o4-" use OpenAI directly
+OPENROUTER_API_KEY=your_openrouter_key
+ANTHROPIC_API_KEY=your_anthropic_key
+OPENAI_API_KEY=your_openai_key

.gitignore

@@ -1,6 +1,10 @@
 spd/scripts/sweep_params.yaml
 docs/coverage/**
 notebooks/**
+scratch/
+
+# Script outputs (generated files, often large)
+scripts/outputs/
 
 **/out/
 neuronpedia_outputs/
@@ -173,4 +177,7 @@ cython_debug/
 #.idea/
 
 **/*.db
-**/*.db*
+**/*.db*
+*.schema.json
+
+.claude/worktrees

.mcp.json

@@ -1,8 +1,3 @@
 {
-  "mcpServers": {
-    "svelte-llm": {
-      "type": "http",
-      "url": "https://svelte-llm.stanislav.garden/mcp/mcp"
-    }
-  }
+  "mcpServers": {}
 }

Makefile

@@ -6,8 +6,10 @@ install: copy-templates
 .PHONY: install-dev
 install-dev: copy-templates
 	uv sync
-	pre-commit install
+	uv run pre-commit install
 
+.PHONY: install-all
+install-all: install-dev install-app
 
 # special install for CI (GitHub Actions) that reduces disk usage and install time
 # 1. create a fresh venv with `--clear` -- this is mostly only for local testing of the CI install

pyproject.toml

@@ -30,9 +30,12 @@ dependencies = [
     "scipy>=1.14.1",
     "fastapi",
     "uvicorn",
+    "orjson",
+    "aiolimiter>=1.2",
     "openrouter>=0.1.1",
     "httpx>=0.28.0",
-    "zstandard" # For streaming datasets
+    "zstandard",  # For streaming datasets
+    "kaleido==0.2.1",
 ]
 
 [dependency-groups]
@@ -54,6 +57,9 @@ spd-clustering = "spd.clustering.scripts.run_pipeline:cli"
 spd-harvest = "spd.harvest.scripts.run_slurm_cli:cli"
 spd-autointerp = "spd.autointerp.scripts.run_slurm_cli:cli"
 spd-attributions = "spd.dataset_attributions.scripts.run_slurm_cli:cli"
+spd-investigate = "spd.investigate.scripts.run_slurm_cli:cli"
+spd-postprocess = "spd.postprocess.cli:cli"
+spd-graph-interp = "spd.graph_interp.scripts.run_slurm_cli:cli"
 
 [build-system]
 requires = ["setuptools", "wheel"]

spd/adapters/__init__.py

@@ -0,0 +1,65 @@
+"""Harvest method adapters: method-specific logic for the generic harvest pipeline.
+
+Each decomposition method (SPD, CLT, MOLT, Transcoder) provides an adapter that knows how to:
+- Load the model and build a dataloader
+- Compute firings and activations from a batch (harvest_fn)
+- Report layer structure and vocab size
+
+Construct via adapter_from_config(method_config).
+"""
+
+from spd.adapters.base import DecompositionAdapter
+from spd.harvest.config import DecompositionMethodHarvestConfig
+
+
+def adapter_from_config(method_config: DecompositionMethodHarvestConfig) -> DecompositionAdapter:
+    from spd.harvest.config import (
+        CLTHarvestConfig,
+        SPDHarvestConfig,
+        TranscoderHarvestConfig,
+    )
+
+    match method_config:
+        case SPDHarvestConfig():
+            from spd.adapters.spd import SPDAdapter
+
+            return SPDAdapter(method_config.id)
+        case TranscoderHarvestConfig():
+            from spd.adapters.transcoder import TranscoderAdapter
+
+            return TranscoderAdapter(method_config)
+        case CLTHarvestConfig():
+            from spd.adapters.clt import CLTAdapter
+
+            return CLTAdapter(method_config)
+
+
+def adapter_from_id(decomposition_id: str) -> DecompositionAdapter:
+    """Construct an adapter from a decomposition ID (e.g. "s-abc123", "tc-1a2b3c4d").
+
+    For SPD runs, the ID is sufficient. For other methods, recovers the full
+    method config from the harvest DB (which is always populated before downstream
+    steps like autointerp run).
+    """
+    if decomposition_id.startswith("s-"):
+        from spd.adapters.spd import SPDAdapter
+
+        return SPDAdapter(decomposition_id)
+
+    return adapter_from_config(_load_method_config(decomposition_id))
+
+
+def _load_method_config(decomposition_id: str) -> DecompositionMethodHarvestConfig:
+    from pydantic import TypeAdapter
+
+    from spd.harvest.repo import HarvestRepo
+
+    repo = HarvestRepo.open_most_recent(decomposition_id)
+    assert repo is not None, (
+        f"No harvest data found for {decomposition_id!r}. "
+        f"Run spd-harvest first to populate the method config."
+    )
+    config_dict = repo.get_config()
+    method_config_raw = config_dict["method_config"]
+    ta = TypeAdapter(DecompositionMethodHarvestConfig)
+    return ta.validate_python(method_config_raw)