From fe37d311276604da5e97194faf7df7363f0731af Mon Sep 17 00:00:00 2001 From: bk-ty Date: Fri, 1 May 2026 10:31:18 -0500 Subject: [PATCH 01/51] =?UTF-8?q?feat(health):=20Phase=202=20=E2=80=94=20t?= =?UTF-8?q?rends,=20filtering,=20and=20sorting=20for=20health=20dashboard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add previous_score and previous_check_scores fields to HealthReport struct with serde skip_serializing_if so existing stored JSON deserializes cleanly - Populate those fields in compute_health by fetching the latest stored report before writing the new one (enables per-check and overall score trending) - Export getTrend helper from HealthCheckRow.tsx; add trend and severityBadge optional props to HealthCheckRowProps; render trend arrow and severity emoji in the check row header - Add FilterState type with severity/fixable/sort dimensions and DEFAULT_FILTER - Add getSeverityBadge and getVisibleChecks helpers to HealthWidget.tsx - Wire filter state into HealthPanel: issueChecks now uses getVisibleChecks, filter bar (severity / fixable / sort + Clear button) shown above check list - Overall score header shows trend arrow coloured green/red/grey based on delta - HealthCheckRow render passes trend and severityBadge from report data Verified: cargo check -p atomic-core, cargo check -p atomic-server, npx tsc --noEmit all pass --- crates/atomic-core/src/briefing/mod.rs | 17 +- crates/atomic-core/src/db.rs | 40 + crates/atomic-core/src/health/audit.rs | 200 +++ crates/atomic-core/src/health/checks.rs | 418 ++++++ crates/atomic-core/src/health/fixes.rs | 545 +++++++ .../atomic-core/src/health/link_resolution.rs | 510 +++++++ crates/atomic-core/src/health/llm_fixes.rs | 226 +++ crates/atomic-core/src/health/mod.rs | 658 +++++++++ crates/atomic-core/src/health/task.rs | 107 ++ crates/atomic-core/src/lib.rs | 45 + crates/atomic-core/src/models.rs | 4 + crates/atomic-core/src/settings.rs | 9 + crates/atomic-core/src/storage/mod.rs | 204 +++ .../atomic-core/src/storage/postgres/wiki.rs | 40 +- .../atomic-core/src/storage/sqlite/health.rs | 798 +++++++++++ crates/atomic-core/src/storage/sqlite/mod.rs | 1 + crates/atomic-core/src/wiki/mod.rs | 35 +- crates/atomic-server/src/lib.rs | 23 + crates/atomic-server/src/main.rs | 3 + crates/atomic-server/src/routes/health.rs | 224 +++ crates/atomic-server/src/routes/import.rs | 18 +- crates/atomic-server/src/routes/mod.rs | 15 +- .../manual/guides/tag-accordion-scroll-fix.md | 117 ++ .../plan.md | 1248 +++++++++++++++++ .../2026-04-30-tag-accordion-scroll-issue.md | 182 +++ ...0-wiki-generate-update-heading-mismatch.md | 291 ++++ .../REVIEW.md | 239 ++++ .../plan.md | 972 +++++++++++++ src/components/dashboard/registry.ts | 14 +- .../dashboard/widgets/HealthCheckRow.tsx | 169 +++ .../dashboard/widgets/HealthReviewModal.tsx | 455 ++++++ .../dashboard/widgets/HealthWidget.tsx | 597 ++++++++ .../dashboard/widgets/NewWikisWidget.tsx | 12 +- .../dashboard/widgets/RecentWikisWidget.tsx | 46 + .../dashboard/widgets/RevisionsWidget.tsx | 31 +- src/components/wiki/WikiCard.tsx | 3 +- src/components/wiki/WikiReader.tsx | 8 +- src/lib/transport/command-map.ts | 36 +- src/stores/wiki.ts | 2 + 39 files changed, 8516 insertions(+), 46 deletions(-) create mode 100644 crates/atomic-core/src/health/audit.rs create mode 100644 crates/atomic-core/src/health/checks.rs create mode 100644 crates/atomic-core/src/health/fixes.rs create mode 100644 crates/atomic-core/src/health/link_resolution.rs create mode 100644 crates/atomic-core/src/health/llm_fixes.rs create mode 100644 crates/atomic-core/src/health/mod.rs create mode 100644 crates/atomic-core/src/health/task.rs create mode 100644 crates/atomic-core/src/storage/sqlite/health.rs create mode 100644 crates/atomic-server/src/routes/health.rs create mode 100644 docs/manual/guides/tag-accordion-scroll-fix.md create mode 100644 docs/plans/2026-04-30-knowledge-health-dashboard/plan.md create mode 100644 docs/plans/2026-04-30-tag-accordion-scroll-issue.md create mode 100644 docs/plans/2026-04-30-wiki-generate-update-heading-mismatch.md create mode 100644 docs/plans/2026-05-01-health-dashboard-ui-improvements/REVIEW.md create mode 100644 docs/plans/2026-05-01-health-dashboard-ui-improvements/plan.md create mode 100644 src/components/dashboard/widgets/HealthCheckRow.tsx create mode 100644 src/components/dashboard/widgets/HealthReviewModal.tsx create mode 100644 src/components/dashboard/widgets/HealthWidget.tsx create mode 100644 src/components/dashboard/widgets/RecentWikisWidget.tsx diff --git a/crates/atomic-core/src/briefing/mod.rs b/crates/atomic-core/src/briefing/mod.rs index 35d8f14d..adc9eafc 100644 --- a/crates/atomic-core/src/briefing/mod.rs +++ b/crates/atomic-core/src/briefing/mod.rs @@ -103,10 +103,25 @@ pub async fn run_briefing( } // Run the agent loop. - let (content, citations) = agentic::generate(core, &since, &new_atoms, total_new) + let (mut content, citations) = agentic::generate(core, &since, &new_atoms, total_new) .await .map_err(AtomicCoreError::Wiki)?; + // Append health summary when score is concerning + if let Ok(report) = crate::health::compute_health(core).await { + if report.overall_score < 85 { + let health_section = format!( + "\n\n## Knowledge Health\n\n\ + Your knowledge base health score is **{}/100** ({}).\n\n\ + {} issues can be auto-fixed via the dashboard.", + report.overall_score, + report.overall_status, + report.auto_fixable + ); + content.push_str(&health_section); + } + } + let id = uuid::Uuid::new_v4().to_string(); let now = Utc::now().to_rfc3339(); let briefing = Briefing { diff --git a/crates/atomic-core/src/db.rs b/crates/atomic-core/src/db.rs index a1d37f03..f3918398 100644 --- a/crates/atomic-core/src/db.rs +++ b/crates/atomic-core/src/db.rs @@ -816,6 +816,46 @@ impl Database { conn.execute_batch("PRAGMA user_version = 16;")?; } + // --- V16 → V17: Knowledge health tables --- + if version < 17 { + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS health_reports ( + id TEXT PRIMARY KEY, + computed_at TEXT NOT NULL, + overall_score INTEGER NOT NULL, + check_scores TEXT NOT NULL, + atom_count INTEGER NOT NULL, + auto_fixes_applied INTEGER NOT NULL DEFAULT 0, + report_json TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_health_reports_computed + ON health_reports(computed_at DESC); + + CREATE TABLE IF NOT EXISTS health_fix_log ( + id TEXT PRIMARY KEY, + check_name TEXT NOT NULL, + action TEXT NOT NULL, + tier TEXT NOT NULL, + atom_ids TEXT, + tag_ids TEXT, + before_state TEXT NOT NULL DEFAULT '{}', + after_state TEXT NOT NULL DEFAULT '{}', + llm_prompt TEXT, + llm_response TEXT, + executed_at TEXT NOT NULL, + undone_at TEXT + ); + CREATE INDEX IF NOT EXISTS idx_health_fix_log_executed + ON health_fix_log(executed_at DESC); + CREATE INDEX IF NOT EXISTS idx_health_fix_log_check + ON health_fix_log(check_name); + + PRAGMA user_version = 17; + "#, + )?; + } + // --- Triggers (recreated every startup to stay current) --- conn.execute_batch( "DROP TRIGGER IF EXISTS atom_tags_insert_count; diff --git a/crates/atomic-core/src/health/audit.rs b/crates/atomic-core/src/health/audit.rs new file mode 100644 index 00000000..6cc0c9ed --- /dev/null +++ b/crates/atomic-core/src/health/audit.rs @@ -0,0 +1,200 @@ +//! Fix audit log and undo capability. +//! +//! Every auto-fix action is recorded in `health_fix_log` with a JSON +//! `before_state` snapshot. `undo_fix` reads that snapshot and restores the +//! affected atoms / tags. + +use crate::error::AtomicCoreError; +use crate::AtomicCore; +use serde::{Deserialize, Serialize}; + +/// A persisted record of one fix action (stored in `health_fix_log`). +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthFixLog { + pub id: String, + pub check_name: String, + pub action: String, + /// "safe" | "low" | "medium" | "high" + pub tier: String, + /// JSON array of atom IDs touched by this fix. + pub atom_ids: Option>, + /// JSON array of tag IDs touched by this fix. + pub tag_ids: Option>, + /// Full JSON snapshot before the fix was applied. Used by undo. + pub before_state: String, + /// Full JSON snapshot after the fix was applied. + pub after_state: String, + pub llm_prompt: Option, + pub llm_response: Option, + pub executed_at: String, + pub undone_at: Option, +} + +/// Lightweight record stored in `health_reports`. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StoredHealthReport { + pub id: String, + pub computed_at: String, + pub overall_score: u32, + /// JSON map check_name → score. + pub check_scores: String, + pub atom_count: i32, + pub auto_fixes_applied: i32, + /// Full serialised `HealthReport`. + pub report_json: String, +} + +/// Record a fix action in the audit log, returning the generated `id`. +pub async fn log_fix( + core: &AtomicCore, + check_name: &str, + action: &str, + tier: &str, + atom_ids: Option<&[String]>, + tag_ids: Option<&[String]>, + before_state: serde_json::Value, + after_state: serde_json::Value, + llm_prompt: Option<&str>, + llm_response: Option<&str>, +) -> Result { + let id = uuid::Uuid::new_v4().to_string(); + let log = HealthFixLog { + id: id.clone(), + check_name: check_name.to_string(), + action: action.to_string(), + tier: tier.to_string(), + atom_ids: atom_ids.map(|ids| ids.to_vec()), + tag_ids: tag_ids.map(|ids| ids.to_vec()), + before_state: serde_json::to_string(&before_state).unwrap_or_default(), + after_state: serde_json::to_string(&after_state).unwrap_or_default(), + llm_prompt: llm_prompt.map(|s| s.to_string()), + llm_response: llm_response.map(|s| s.to_string()), + executed_at: chrono::Utc::now().to_rfc3339(), + undone_at: None, + }; + core.storage().log_fix_action_sync(&log).await?; + Ok(id) +} + +/// Undo a previously applied fix using the stored `before_state` snapshot. +/// +/// Currently supports: +/// - Recreating deleted atoms (JSON array of `{id, content, source_url, tags}`) +/// - Recreating deleted tags (JSON array of `{id, name, parent_id}`) +/// - Restoring updated atom content (JSON array of `{id, content}`) +pub async fn undo(core: &AtomicCore, fix_id: &str) -> Result<(), AtomicCoreError> { + let log = core + .storage() + .get_fix_log_sync(fix_id) + .await? + .ok_or_else(|| { + AtomicCoreError::NotFound(format!("health fix log {fix_id} not found")) + })?; + + if log.undone_at.is_some() { + return Err(AtomicCoreError::Validation(format!( + "fix {fix_id} has already been undone" + ))); + } + + let before: serde_json::Value = serde_json::from_str(&log.before_state) + .unwrap_or(serde_json::json!({})); + + match log.action.as_str() { + "deleted_tags" => { + // before_state: [ { "id": "...", "name": "...", "parent_id": null } ] + if let Some(tags) = before.as_array() { + for tag in tags { + let name = tag + .get("name") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + let parent_id = tag.get("parent_id").and_then(|v| v.as_str()); + // Re-create tag (ID won't be preserved, but name + parent will match) + if !name.is_empty() { + let _ = core.storage().create_tag_impl(name, parent_id).await; + } + } + } + } + "deleted_atoms" => { + // before_state: [ { "id": "...", "content": "...", "source_url": null, "tag_ids": [...] } ] + if let Some(atoms) = before.as_array() { + for atom_snap in atoms { + let content = atom_snap + .get("content") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + let source_url = atom_snap + .get("source_url") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let tag_ids: Vec = atom_snap + .get("tag_ids") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(|s| s.to_string())) + .collect() + }) + .unwrap_or_default(); + let req = crate::CreateAtomRequest { + content, + source_url, + tag_ids, + ..Default::default() + }; + let _ = core.create_atom(req, |_| {}).await; + } + } + } + "updated_atoms" => { + // before_state: [ { "id": "...", "content": "...", "source_url": null } ] + if let Some(atoms) = before.as_array() { + for atom_snap in atoms { + let id = atom_snap + .get("id") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + let content = atom_snap + .get("content") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + let source_url = atom_snap + .get("source_url") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + if !id.is_empty() { + let upd = crate::UpdateAtomRequest { + content, + source_url, + published_at: None, + tag_ids: None, + }; + let _ = core.update_atom(&id, upd, |_| {}).await; + } + } + } + } + _ => { + tracing::warn!(action = %log.action, "undo not implemented for this action type"); + } + } + + // Mark fix as undone + core.storage().mark_fix_undone_sync(fix_id).await?; + Ok(()) +} + +/// Fetch recent fix log entries (most recent first). +pub async fn get_recent_fixes( + core: &AtomicCore, + limit: i32, +) -> Result, AtomicCoreError> { + core.storage().get_recent_fixes_sync(limit).await +} diff --git a/crates/atomic-core/src/health/checks.rs b/crates/atomic-core/src/health/checks.rs new file mode 100644 index 00000000..dd720e40 --- /dev/null +++ b/crates/atomic-core/src/health/checks.rs @@ -0,0 +1,418 @@ +//! Individual health check implementations. +//! +//! Each check takes a `&HealthRawData` snapshot (fetched once) and returns a +//! `HealthCheckResult` with a 0–100 score and check-specific JSON data. + +use super::{DuplicatePair, HealthCheckResult, WikiGap, WikiStaleEntry}; +use crate::storage::sqlite::health::HealthRawData; +use serde_json::json; +use std::collections::HashMap; + +/// Run all 10 checks against pre-fetched raw data. +pub fn run_all(raw: &HealthRawData) -> HashMap { + let mut map = HashMap::new(); + map.insert("embedding_coverage".to_string(), embedding_coverage(raw)); + map.insert("tagging_coverage".to_string(), tagging_coverage(raw)); + map.insert("source_uniqueness".to_string(), source_uniqueness(raw)); + map.insert("orphan_tags".to_string(), orphan_tags(raw)); + map.insert( + "semantic_graph_freshness".to_string(), + semantic_graph_freshness(raw), + ); + map.insert("wiki_coverage".to_string(), wiki_coverage(raw)); + map.insert("content_quality".to_string(), content_quality(raw)); + map.insert("tag_health".to_string(), tag_health(raw)); + map.insert("content_overlap".to_string(), content_overlap(raw)); + map.insert( + "contradiction_detection".to_string(), + contradiction_detection(raw), + ); + // Diagnostic check — not included in CHECK_WEIGHTS, doesn't affect score. + // Surfaces boilerplate-dominated atoms to the UI without penalising the KB. + map.insert( + "boilerplate_pollution".to_string(), + boilerplate_pollution(raw), + ); + map +} + +// ==================== Individual checks ==================== + +pub fn embedding_coverage(raw: &HealthRawData) -> HealthCheckResult { + let total = raw.total_atoms; + let complete = raw.embedding_complete; + let pending = raw.embedding_pending; + let processing = raw.embedding_processing; + let failed = raw.embedding_failed; + + let score = if total == 0 { + 100 + } else { + let pct = (complete as f64 / total as f64 * 100.0) as u32; + if failed > 0 { + pct.min(50) + } else { + pct + } + }; + let status = if score == 100 { + "ok" + } else if score >= 80 { + "warning" + } else { + "error" + }; + HealthCheckResult { + status: status.to_string(), + score, + auto_fixable: failed > 0 || pending > 0, + requires_review: false, + fix_action: Some("retry_failed_and_process_pending".to_string()), + data: json!({ + "total": total, + "complete": complete, + "pending": pending, + "processing": processing, + "failed": failed + }), + } +} + +pub fn tagging_coverage(raw: &HealthRawData) -> HealthCheckResult { + let total = raw.total_atoms; + let failed = raw.tagging_failed; + let pending = raw.tagging_pending; + let untagged = raw.untagged_complete; + // skipped_untagged: the tagger skipped these atoms AND they have 0 tags. + // Atoms with tagging_status='skipped' that DO have tags are fine — + // they were imported with existing tags and the tagger deliberately skipped. + let skipped_untagged = raw.skipped_untagged; + + // Only count actually-problematic atoms: failed, truly pending, complete-but-untagged, + // and skipped-with-no-tags. Skipped atoms that HAVE tags are fine. + let bad = (failed + pending + untagged + skipped_untagged).min(total); + let tagged = (total - bad).max(0); + + let score = if total == 0 { + 100 + } else { + (tagged as f64 / total as f64 * 100.0) as u32 + }; + let status = if score == 100 { + "ok" + } else if score >= 80 { + "warning" + } else { + "error" + }; + HealthCheckResult { + status: status.to_string(), + score, + auto_fixable: failed > 0 || pending > 0 || untagged > 0 || skipped_untagged > 0, + requires_review: false, + fix_action: Some("retry_tagging_pipeline".to_string()), + data: json!({ + "total": total, + "tagged": tagged, + "untagged_complete": untagged, + "skipped_untagged": skipped_untagged, + "failed": failed, + "pending": pending, + "skipped_with_tags": raw.tagging_skipped - skipped_untagged + }), + } +} + +pub fn source_uniqueness(raw: &HealthRawData) -> HealthCheckResult { + let dup_count = raw.duplicate_sources.len() as i32; + let score = (100i32 - dup_count * 15).max(0) as u32; + let status = if dup_count == 0 { "ok" } else { "warning" }; + let pairs: Vec = raw + .duplicate_sources + .iter() + .map(|(url, ids)| { + json!({ + "source_url": url, + "atom_count": ids.len(), + "atom_ids": ids + }) + }) + .collect(); + HealthCheckResult { + status: status.to_string(), + score, + auto_fixable: dup_count > 0, + requires_review: false, + fix_action: Some("merge_exact_source_duplicates".to_string()), + data: json!({ + "count": dup_count, + "pairs": pairs + }), + } +} + +pub fn orphan_tags(raw: &HealthRawData) -> HealthCheckResult { + let count = raw.orphan_tags.len() as i32; + let score = (100i32 - count * 2).max(0) as u32; + let status = if count == 0 { "ok" } else { "warning" }; + let tag_list: Vec = raw + .orphan_tags + .iter() + .map(|(id, name)| json!({ "id": id, "name": name })) + .collect(); + HealthCheckResult { + status: status.to_string(), + score, + auto_fixable: count > 0, + requires_review: false, + fix_action: Some("delete_orphan_tags".to_string()), + data: json!({ "count": count, "tags": tag_list }), + } +} + +pub fn semantic_graph_freshness(raw: &HealthRawData) -> HealthCheckResult { + let atoms_since = raw.atoms_since_edge_rebuild; + let score = (100i32 - atoms_since * 2).max(0) as u32; + let status = if atoms_since == 0 { + "ok" + } else if atoms_since <= 20 { + "warning" + } else { + "error" + }; + HealthCheckResult { + status: status.to_string(), + score, + auto_fixable: atoms_since > 0, + requires_review: false, + fix_action: Some("rebuild_semantic_edges".to_string()), + data: json!({ + "last_rebuilt": raw.newest_edge_created_at, + "newest_atom": raw.newest_atom_updated_at, + "atoms_since_rebuild": atoms_since + }), + } +} + +pub fn wiki_coverage(raw: &HealthRawData) -> HealthCheckResult { + let eligible = raw.wiki_eligible_count; + let with_wiki = raw.wiki_present_count; + let stale = raw.wiki_stale_count; + let without_wiki = eligible - with_wiki; + + let score = if eligible == 0 { + 100 + } else { + let coverage_pct = (with_wiki as f64 / eligible as f64) * 70.0; + let freshness_pct = if with_wiki == 0 { + 30.0 + } else { + let non_stale = (with_wiki - stale).max(0); + (non_stale as f64 / with_wiki as f64) * 30.0 + }; + (coverage_pct + freshness_pct).round() as u32 + }; + let status = if score >= 90 { + "ok" + } else if score >= 60 { + "warning" + } else { + "error" + }; + + let gaps: Vec = raw + .wiki_gaps + .iter() + .map(|g: &WikiGap| json!({ "tag_id": g.tag_id, "tag_name": g.tag_name, "atom_count": g.atom_count })) + .collect(); + let stale_list: Vec = raw + .wiki_stale + .iter() + .map(|s: &WikiStaleEntry| { + json!({ "tag_id": s.tag_id, "tag_name": s.tag_name, "new_atoms": s.new_atom_count }) + }) + .collect(); + + HealthCheckResult { + status: status.to_string(), + score, + auto_fixable: without_wiki > 0 || stale > 0, + requires_review: false, + fix_action: Some("generate_missing_wikis".to_string()), + data: json!({ + "eligible_tags": eligible, + "with_wiki": with_wiki, + "without_wiki": without_wiki, + "stale_wikis": stale, + "gaps": gaps, + "stale": stale_list + }), + } +} + +pub fn content_quality(raw: &HealthRawData) -> HealthCheckResult { + let mut issues = 0; + if !raw.very_short_atoms.is_empty() { + issues += 1; + } + if !raw.very_long_atoms.is_empty() { + issues += 1; + } + if !raw.no_heading_atoms.is_empty() { + issues += 1; + } + if !raw.no_source_atoms.is_empty() { + issues += 1; + } + + let score = (85u32).saturating_sub(issues * 5); + let status = if issues == 0 { "ok" } else { "info" }; + + HealthCheckResult { + status: status.to_string(), + score, + auto_fixable: !raw.very_short_atoms.is_empty() + || !raw.very_long_atoms.is_empty() + || !raw.no_heading_atoms.is_empty(), + requires_review: !raw.no_source_atoms.is_empty(), + fix_action: None, + data: json!({ + "total": raw.total_atoms, + "issues": { + "very_short": { + "count": raw.very_short_atoms.len(), + "auto_fixable": true, + "atoms": raw.very_short_atoms + }, + "very_long": { + "count": raw.very_long_atoms.len(), + "auto_fixable": true, + "atoms": raw.very_long_atoms + }, + "no_headings": { + "count": raw.no_heading_atoms.len(), + "auto_fixable": true, + "atoms": raw.no_heading_atoms + }, + "no_source": { + "count": raw.no_source_atoms.len(), + "auto_fixable": false, + "atoms": raw.no_source_atoms + } + } + }), + } +} + +pub fn tag_health(raw: &HealthRawData) -> HealthCheckResult { + let single = raw.single_atom_tags; + let rootless = raw.rootless_tags; + let similar = raw.similar_name_pair_count; + + let issues = (single > 3) as u32 + (rootless > 0) as u32 + (similar > 0) as u32; + let score = (100u32).saturating_sub(issues * 10); + let status = if issues == 0 { "ok" } else { "warning" }; + + HealthCheckResult { + status: status.to_string(), + score, + auto_fixable: similar > 0, + requires_review: rootless > 0, + fix_action: None, + data: json!({ + "single_atom_tags": single, + "rootless_tags": rootless, + "similar_name_pairs": similar + }), + } +} + +pub fn content_overlap(raw: &HealthRawData) -> HealthCheckResult { + let overlaps = raw.duplicate_pairs.len() as i32; + let exact_dupes = raw.duplicate_sources.len() as i32; + let template_clones = raw.boilerplate_affected_atoms.len() as i32; + + // Score: deduct for unreviewed cross-source overlaps. + // Exact dupes handled by source_uniqueness, template clones by boilerplate_pollution. + let score = (100i32 - overlaps * 8).max(0) as u32; + let status = if overlaps == 0 { "ok" } else { "warning" }; + + let pairs: Vec = raw + .duplicate_pairs + .iter() + .map(|p: &DuplicatePair| { + json!({ + "pair_id": p.pair_id, + "atom_a": { "id": p.atom_a_id, "title": p.atom_a_title, "source": p.atom_a_source }, + "atom_b": { "id": p.atom_b_id, "title": p.atom_b_title, "source": p.atom_b_source }, + "similarity": p.similarity, + "shared_tag_count": p.shared_tag_count, + "available_actions": ["merge_with_llm", "keep_both", "delete_older", "mark_complementary"] + }) + }) + .collect(); + + HealthCheckResult { + status: status.to_string(), + score, + auto_fixable: false, + requires_review: overlaps > 0, + fix_action: None, + data: json!({ + "exact_duplicates": exact_dupes, + "template_clones": template_clones, + "cross_source_overlaps": overlaps, + "count": overlaps, + "pairs": pairs + }), + } +} + +pub fn contradiction_detection(raw: &HealthRawData) -> HealthCheckResult { + let count = raw.contradiction_candidate_count; + let score = (100i32 - count * 10).max(0) as u32; + let status = if count == 0 { "ok" } else { "warning" }; + + HealthCheckResult { + status: status.to_string(), + score, + auto_fixable: false, + requires_review: count > 0, + fix_action: None, + data: json!({ + "pairs_checked": raw.contradiction_pairs_checked, + "potential_contradictions": count + }), + } +} + + +/// Diagnostic check: atoms whose embeddings are dominated by shared boilerplate. +/// +/// An atom is flagged when it has >= 2 semantic edges at similarity >= 0.99. +/// That means the vector space can't distinguish it from multiple other atoms, +/// so semantic search will return the wrong runbook / article for those queries. +/// +/// This check does NOT affect the overall score (not in CHECK_WEIGHTS). +/// Fix: re-chunk excluding boilerplate sections, or re-embed with a unique-content prefix. +pub fn boilerplate_pollution(raw: &HealthRawData) -> HealthCheckResult { + let count = raw.boilerplate_affected_atoms.len() as i32; + let status = if count == 0 { "ok" } else { "warning" }; + + HealthCheckResult { + status: status.to_string(), + // Always 100 — diagnostic only, does not affect overall KB score. + score: 100, + auto_fixable: false, + requires_review: count > 0, + fix_action: None, + data: json!({ + "count": count, + "affected_atoms": raw.boilerplate_affected_atoms, + "description": "Atoms with >= 2 near-identical edges (similarity >= 0.99). \ + Shared boilerplate text drowns out unique content in their \ + embeddings. Semantic search cannot reliably distinguish \ + these atoms from each other." + }), + } +} \ No newline at end of file diff --git a/crates/atomic-core/src/health/fixes.rs b/crates/atomic-core/src/health/fixes.rs new file mode 100644 index 00000000..49728a3b --- /dev/null +++ b/crates/atomic-core/src/health/fixes.rs @@ -0,0 +1,545 @@ +//! Deterministic (non-LLM) auto-fix implementations. +//! +//! Each function either executes the fix immediately (if `dry_run = false`) or +//! describes what it would do (if `dry_run = true`). Every executed fix logs +//! a `HealthFixLog` row for undo support. + +use super::{audit, FixAction}; +use crate::error::AtomicCoreError; +use crate::storage::sqlite::health::HealthRawData; +use crate::AtomicCore; +use serde_json::json; + + +/// Retry failed embeddings and process pending ones. Safe tier. +pub async fn fix_embedding_coverage( + core: &AtomicCore, + dry_run: bool, +) -> Result, AtomicCoreError> { + let status = core.get_pipeline_status().await?; + let pending = status.pending; + let failed = status.failed_count; + let count = pending + failed; + + if count == 0 { + return Ok(None); + } + + let id = if dry_run { + "dry_run".to_string() + } else { + let retried = core.retry_failed_embeddings(|_| {}).await.unwrap_or(0); + let processed = core.process_pending_embeddings(|_| {}).await.unwrap_or(0); + tracing::info!(retried, processed, "embedding_coverage fix applied"); + + audit::log_fix( + core, + "embedding_coverage", + "retry_failed_and_process_pending", + "safe", + None, + None, + json!({"failed": failed, "pending": pending}), + json!({"retried": retried, "processed": processed}), + None, + None, + ) + .await? + }; + + Ok(Some(FixAction { + id, + check: "embedding_coverage".to_string(), + action: "retry_failed_and_process_pending".to_string(), + count, + details: vec![ + format!("{} failed retried", failed), + format!("{} pending processed", pending), + ], + })) +} + +/// Queue a semantic edge graph rebuild. Safe tier. +pub async fn fix_graph_freshness( + core: &AtomicCore, + dry_run: bool, +) -> Result, AtomicCoreError> { + let id = if dry_run { + "dry_run".to_string() + } else { + let edges = core.rebuild_semantic_edges().await.unwrap_or(0); + tracing::info!(edges, "semantic_graph_freshness fix: edges rebuilt"); + + audit::log_fix( + core, + "semantic_graph_freshness", + "queued_rebuild", + "safe", + None, + None, + json!({}), + json!({"edges_rebuilt": edges}), + None, + None, + ) + .await? + }; + + Ok(Some(FixAction { + id, + check: "semantic_graph_freshness".to_string(), + action: "queued_rebuild".to_string(), + count: 1, + details: vec!["Semantic edge graph rebuild queued".to_string()], + })) +} + +/// Reset skipped-with-no-tags atoms to pending and run the tagging pipeline. Safe tier. +/// +/// These are atoms whose `tagging_status = 'skipped'` AND have zero tags assigned. +/// They were typically imported before auto-tagging was configured and never retried. +pub async fn fix_tagging_coverage( + core: &AtomicCore, + skipped_untagged_count: i32, + dry_run: bool, +) -> Result, AtomicCoreError> { + if skipped_untagged_count == 0 { + return Ok(None); + } + + let id = if dry_run { + "dry_run".to_string() + } else { + let reset = core + .storage() + .reset_skipped_untagged_to_pending_sync() + .await + .unwrap_or(0); + let processed = core.process_pending_tagging(|_| {}).await.unwrap_or(0); + tracing::info!(reset, processed, "tagging_coverage fix: skipped atoms re-queued"); + + audit::log_fix( + core, + "tagging_coverage", + "reset_skipped_untagged_to_pending", + "safe", + None, + None, + json!({"skipped_untagged": skipped_untagged_count}), + json!({"reset": reset, "processed": processed}), + None, + None, + ) + .await? + }; + + Ok(Some(FixAction { + id, + check: "tagging_coverage".to_string(), + action: "reset_skipped_untagged_to_pending".to_string(), + count: skipped_untagged_count, + details: vec![format!("{} atoms reset to pending for re-tagging", skipped_untagged_count)], + })) +} + +/// Delete orphan tags (tags with 0 atoms and no children). Low tier. +pub async fn fix_orphan_tags( + core: &AtomicCore, + raw: &HealthRawData, + dry_run: bool, +) -> Result, AtomicCoreError> { + if raw.orphan_tags.is_empty() { + return Ok(None); + } + + let count = raw.orphan_tags.len() as i32; + let names: Vec = raw.orphan_tags.iter().map(|(_, n)| n.clone()).collect(); + let ids: Vec = raw.orphan_tags.iter().map(|(id, _)| id.clone()).collect(); + + let before_state = json!(raw + .orphan_tags + .iter() + .map(|(id, name)| json!({"id": id, "name": name, "parent_id": null})) + .collect::>()); + + let id = if dry_run { + "dry_run".to_string() + } else { + for tag_id in &ids { + if let Err(e) = core.delete_tag(tag_id, false).await { + tracing::warn!(tag_id, error = %e, "failed to delete orphan tag"); + } + } + tracing::info!(count, "orphan_tags fix: deleted tags"); + + audit::log_fix( + core, + "orphan_tags", + "deleted_tags", + "low", + None, + Some(&ids), + before_state, + json!({"deleted": count}), + None, + None, + ) + .await? + }; + + Ok(Some(FixAction { + id, + check: "orphan_tags".to_string(), + action: "deleted_tags".to_string(), + count, + details: names, + })) +} + +/// Generate missing wiki articles for eligible tags. Low tier. +/// Rate-limited to 3 generations per fix run to avoid long waits. +pub async fn fix_wiki_coverage( + core: &AtomicCore, + raw: &HealthRawData, + dry_run: bool, +) -> Result, AtomicCoreError> { + let gaps = &raw.wiki_gaps; + let stale = &raw.wiki_stale; + + if gaps.is_empty() && stale.is_empty() { + return Ok(None); + } + + // Prioritise by atom count (highest first), max 3 total + let mut to_generate: Vec<(String, String)> = gaps + .iter() + .map(|g| (g.tag_id.clone(), g.tag_name.clone())) + .collect(); + // Then stale wikis + for s in stale { + to_generate.push((s.tag_id.clone(), s.tag_name.clone())); + } + to_generate.truncate(3); + + let count = to_generate.len() as i32; + let detail_names: Vec = to_generate.iter().map(|(_, n)| n.clone()).collect(); + + let id = if dry_run { + "dry_run".to_string() + } else { + for (tag_id, tag_name) in &to_generate { + match core.generate_wiki(tag_id, tag_name).await { + Ok(_) => tracing::info!(tag_id, "wiki generated"), + Err(e) => tracing::warn!(tag_id, error = %e, "wiki generation failed"), + } + } + + audit::log_fix( + core, + "wiki_coverage", + "generated_wikis", + "low", + None, + None, + json!({"gaps": gaps.len(), "stale": stale.len()}), + json!({"generated": count}), + None, + None, + ) + .await? + }; + + Ok(Some(FixAction { + id, + check: "wiki_coverage".to_string(), + action: "generated_wikis".to_string(), + count, + details: detail_names, + })) +} + +/// Deduplicate atoms with the exact same source_url. Medium tier. +/// Keeps newest; merges tags from all duplicates; deletes older copies. +pub async fn fix_source_uniqueness( + core: &AtomicCore, + raw: &HealthRawData, + dry_run: bool, +) -> Result, AtomicCoreError> { + if raw.duplicate_sources.is_empty() { + return Ok(None); + } + + let mut deleted_ids: Vec = Vec::new(); + let mut before_atoms: Vec = Vec::new(); + + for (source_url, atom_ids) in &raw.duplicate_sources { + if atom_ids.len() < 2 { + continue; + } + + // Fetch all atoms in this group to find newest + let mut atoms_with_dates: Vec<(String, String)> = Vec::new(); // (id, updated_at) + for id in atom_ids { + if let Ok(Some(a)) = core.get_atom(id).await { + atoms_with_dates.push((a.atom.id.clone(), a.atom.updated_at.clone())); + + // Capture before state + let tag_ids: Vec = a.tags.iter().map(|t| t.id.clone()).collect(); + before_atoms.push(json!({ + "id": a.atom.id, + "content": a.atom.content, + "source_url": a.atom.source_url, + "tag_ids": tag_ids, + })); + } + } + + // Sort by updated_at desc — newest first + atoms_with_dates.sort_by(|a, b| b.1.cmp(&a.1)); + let keep_id = atoms_with_dates[0].0.clone(); + let to_delete: Vec = atoms_with_dates[1..].iter().map(|(id, _)| id.clone()).collect(); + + if dry_run { + tracing::info!( + source_url, + keep = %keep_id, + delete = ?to_delete, + "dry_run: would merge source duplicates" + ); + } else { + // Collect all tags from duplicates into the keeper + let mut all_tag_ids: std::collections::HashSet = std::collections::HashSet::new(); + for id in &to_delete { + if let Ok(tag_ids) = core.storage().get_atom_tag_ids_impl(id).await { + all_tag_ids.extend(tag_ids); + } + } + // Merge tags onto keeper + if !all_tag_ids.is_empty() { + let tag_list: Vec = all_tag_ids.into_iter().collect(); + let _ = core + .storage() + .link_tags_to_atom_impl(&keep_id, &tag_list) + .await; + } + // Delete duplicates + for id in &to_delete { + if let Err(e) = core.delete_atom(id).await { + tracing::warn!(id, error = %e, "failed to delete source duplicate atom"); + } else { + deleted_ids.push(id.clone()); + } + } + } + } + + if deleted_ids.is_empty() && !dry_run { + return Ok(None); + } + + let count = if dry_run { + raw.duplicate_sources + .iter() + .map(|(_, ids)| (ids.len() as i32 - 1).max(0)) + .sum::() + } else { + deleted_ids.len() as i32 + }; + + let id = if dry_run { + "dry_run".to_string() + } else { + audit::log_fix( + core, + "source_uniqueness", + "deleted_atoms", + "medium", + Some(&deleted_ids), + None, + serde_json::Value::Array(before_atoms), + json!({"deleted": count}), + None, + None, + ) + .await? + }; + + Ok(Some(FixAction { + id, + check: "source_uniqueness".to_string(), + action: "deleted_atoms".to_string(), + count, + details: if dry_run { + raw.duplicate_sources + .iter() + .map(|(url, _)| url.clone()) + .collect() + } else { + deleted_ids + }, + })) +} + + +/// Resolve broken internal links in all atoms to `atom://id` URIs. Medium tier. +/// +/// For each atom with relative markdown links or `[[wikilinks]]`: +/// 1. Resolve the href to a candidate source URL using the atom's vault prefix. +/// 2. Look up the target atom by source URL. +/// 3. Replace the original href with `atom://target_id`. +/// +/// Unresolvable links are left untouched and reported in `details`. +pub async fn fix_broken_internal_links( + core: &AtomicCore, + dry_run: bool, +) -> Result, AtomicCoreError> { + use crate::health::link_resolution::{ + apply_link_replacements, extract_internal_links, vault_root, ResolvedLink, + }; + + let candidates = core.storage().get_link_candidate_atoms_sync().await?; + if candidates.is_empty() { + return Ok(None); + } + + let mut fixed_total = 0i32; + let mut unresolvable: Vec = Vec::new(); + let mut before_state: Vec = Vec::new(); + let mut atom_ids_changed: Vec = Vec::new(); + + for (atom_id, content, source_url) in &candidates { + let links = extract_internal_links(content, source_url.as_deref()); + if links.is_empty() { + continue; + } + + let candidate_urls: Vec = links + .iter() + .flat_map(|l| l.candidate_source_urls.iter().cloned()) + .collect(); + + let url_map = core + .storage() + .find_atoms_by_source_urls_sync(candidate_urls) + .await + .unwrap_or_default(); + + let vault_pfx = source_url + .as_deref() + .and_then(vault_root) + .map(|s| s.to_string()); + + let mut resolved: Vec = Vec::new(); + + for link in &links { + // Try exact source URL match first + let target_id = link + .candidate_source_urls + .iter() + .find_map(|u| url_map.get(u).cloned()); + + let target_id = if target_id.is_none() { + // For wikilinks: fall back to vault-wide name search + if let (Some(name), Some(pfx)) = (&link.wikilink_name, &vault_pfx) { + core.storage() + .find_atom_by_wikilink_name_sync(name.clone(), pfx.clone()) + .await + .unwrap_or(None) + .map(|(id, _)| id) + } else { + None + } + } else { + target_id + }; + + match target_id { + Some(id) => { + resolved.push(ResolvedLink { + original: link.original.clone(), + target_atom_id: id.clone(), + replacement: format!("atom://{}", id), + }); + fixed_total += 1; + } + None => { + unresolvable.push(format!("{} (in {})", link.href, atom_id)); + } + } + } + + if resolved.is_empty() { + continue; + } + + before_state.push(json!({ + "id": atom_id, + "content": content, + "source_url": source_url, + })); + + if !dry_run { + let new_content = apply_link_replacements(content, &resolved); + core.update_atom_content_only(atom_id, crate::UpdateAtomRequest { + content: new_content, + source_url: source_url.clone(), + published_at: None, + tag_ids: None, + }) + .await + .map_err(|e| { + tracing::warn!(atom_id, error = %e, "failed to update atom with resolved links"); + e + })?; + atom_ids_changed.push(atom_id.clone()); + } + } + + if fixed_total == 0 { + return Ok(None); + } + + let id = if dry_run { + "dry_run".to_string() + } else { + audit::log_fix( + core, + "broken_internal_links", + "resolve_internal_links", + "medium", + Some(&atom_ids_changed), + None, + serde_json::Value::Array(before_state), + json!({ + "resolved": fixed_total, + "unresolvable": unresolvable.len(), + }), + None, + None, + ) + .await? + }; + + tracing::info!( + fixed = fixed_total, + unresolvable = unresolvable.len(), + dry_run, + "broken_internal_links fix completed" + ); + + let mut details: Vec = atom_ids_changed + .iter() + .map(|id| format!("Updated: {}", id)) + .collect(); + details.extend( + unresolvable.iter().take(10).map(|s| format!("Unresolvable: {}", s)), + ); + + Ok(Some(FixAction { + id, + check: "broken_internal_links".to_string(), + action: "resolve_internal_links".to_string(), + count: fixed_total, + details, + })) +} \ No newline at end of file diff --git a/crates/atomic-core/src/health/link_resolution.rs b/crates/atomic-core/src/health/link_resolution.rs new file mode 100644 index 00000000..419b8a42 --- /dev/null +++ b/crates/atomic-core/src/health/link_resolution.rs @@ -0,0 +1,510 @@ +//! Generic internal-link extraction and resolution. +//! +//! Handles two link formats found in Obsidian-imported atoms: +//! +//! 1. **Markdown links** — `[text](relative/path.md)` or `[text](../other.md)` +//! 2. **Wikilinks** — `[[File Name]]` or `[[File Name|Display Text]]` +//! +//! A link is "internal" when its href contains no URI scheme (`://`) and is +//! not a bare fragment (`#anchor`). Absolute paths starting with `/` are +//! also excluded — those are server-rooted URLs, not vault-relative paths. +//! +//! Resolution maps each link to a candidate `source_url` (or a set of LIKE +//! patterns for wikilinks) so callers can look the target up in the atom +//! table. + +/// A single internal link found inside an atom's content. +#[derive(Debug, Clone)] +pub struct InternalLink { + /// The original text in the content that needs to be replaced. + /// For markdown: `[text](href)`. For wikilinks: `[[target]]`. + pub original: String, + /// The raw href or wikilink target extracted from the original. + pub href: String, + /// Candidate absolute source URLs to try (exact lookup). + /// Built from the current atom's source_url + relative path. + pub candidate_source_urls: Vec, + /// For wikilinks: search the atoms table with + /// `source_url LIKE '%/' || name || '.md'` across the vault. + pub wikilink_name: Option, +} + +/// A resolved match: an `InternalLink` with its target atom identified. +#[derive(Debug, Clone)] +pub struct ResolvedLink { + pub original: String, + pub target_atom_id: String, + /// Used as the replacement href: `atom://target_atom_id` + pub replacement: String, +} + +// ==================== Extraction ==================== + +/// Extract all internal links from `content`. +/// +/// `source_url` is the current atom's source URL; it is used to resolve +/// relative paths. Pass `None` for atoms without a known source. +pub fn extract_internal_links( + content: &str, + source_url: Option<&str>, +) -> Vec { + let mut links = Vec::new(); + links.extend(extract_markdown_links(content, source_url)); + links.extend(extract_wikilinks(content, source_url)); + links +} + +fn extract_markdown_links(content: &str, source_url: Option<&str>) -> Vec { + let mut links = Vec::new(); + let bytes = content.as_bytes(); + let mut i = 0; + + while i + 1 < bytes.len() { + // Scan for `](` + if bytes[i] != b']' || bytes[i + 1] != b'(' { + i += 1; + continue; + } + + // Find the matching `)` + let href_start = i + 2; + let mut j = href_start; + let mut depth = 1i32; + + while j < bytes.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + if depth > 0 { + j += 1; + } + } + + if depth != 0 { + i += 1; + continue; + } + + let raw_href = match std::str::from_utf8(&bytes[href_start..j]) { + Ok(s) => s, + Err(_) => { + i = j + 1; + continue; + } + }; + + // Strip optional inline title: `path.md "Title"` → `path.md` + let href = raw_href + .trim() + .split('"') + .next() + .unwrap_or("") + .split('\'') + .next() + .unwrap_or("") + .trim() + .to_string(); + + if is_internal_href(&href) && looks_like_document(&href) { + // Find the opening `[` to capture display text + full match + let (original, display) = scan_back_for_display_text(content, i); + + let candidate_source_urls = match source_url { + Some(su) => build_href_candidates(&href, su), + None => vec![], + }; + + links.push(InternalLink { + original, + href: href.clone(), + candidate_source_urls, + wikilink_name: None, + }); + + // Skip `[display](href)` — display text already consumed above + let _ = display; + } + + i = j + 1; + } + + links +} + +fn extract_wikilinks(content: &str, source_url: Option<&str>) -> Vec { + let mut links = Vec::new(); + let bytes = content.as_bytes(); + let mut i = 0; + + while i + 1 < bytes.len() { + if bytes[i] != b'[' || bytes[i + 1] != b'[' { + i += 1; + continue; + } + + let start = i + 2; + let mut j = start; + + while j + 1 < bytes.len() && !(bytes[j] == b']' && bytes[j + 1] == b']') { + j += 1; + } + + if j + 1 >= bytes.len() { + i += 1; + continue; + } + + let inner = match std::str::from_utf8(&bytes[start..j]) { + Ok(s) => s, + Err(_) => { + i = j + 2; + continue; + } + }; + + // `[[target|display text]]` — keep only the target + let target = inner.split('|').next().unwrap_or("").trim().to_string(); + + if !target.is_empty() { + let original = format!("[[{}]]", inner); + + let candidate_source_urls = match source_url { + Some(su) => build_wikilink_exact_candidates(&target, su), + None => vec![], + }; + + links.push(InternalLink { + original, + href: target.clone(), + candidate_source_urls, + wikilink_name: Some(target), + }); + } + + i = j + 2; + } + + links +} + +// ==================== Predicates ==================== + +/// A link is internal when it has no URI scheme, is not a bare fragment, +/// and does not start with `/` (server-root absolute path). +fn is_internal_href(href: &str) -> bool { + let h = href.trim(); + !h.is_empty() + && !h.starts_with('#') + && !h.starts_with('/') + && !h.contains("://") + && !h.starts_with("mailto:") + && !h.starts_with("tel:") +} + +/// The link looks like a document reference (not an image, anchor, etc.). +fn looks_like_document(href: &str) -> bool { + let h = href.trim().to_lowercase(); + // Explicit markdown/text extensions + if h.ends_with(".md") || h.ends_with(".txt") || h.ends_with(".org") { + return true; + } + // Relative path operators + if h.starts_with("./") || h.starts_with("../") { + return true; + } + // No extension + contains path separator → likely a document path + if !h.contains('.') && h.contains('/') { + return true; + } + false +} + +// ==================== URL resolution ==================== + +/// Extract the vault root from a source URL. +/// +/// `obsidian://ar-playbook/some/path.md` → `obsidian://ar-playbook/` +pub fn vault_root(source_url: &str) -> Option<&str> { + let scheme_end = source_url.find("://")?; + let after_scheme = &source_url[scheme_end + 3..]; + let vault_sep = after_scheme.find('/')?; + Some(&source_url[..scheme_end + 3 + vault_sep + 1]) +} + +/// Directory portion of a source URL (everything up to and including the +/// last `/`). +fn source_dir(source_url: &str) -> &str { + if let Some(pos) = source_url.rfind('/') { + &source_url[..pos + 1] + } else { + source_url + } +} + +/// Resolve a relative href against the current atom's source URL, returning +/// candidate source URL strings (with and without `.md`) to try. +fn build_href_candidates(href: &str, current_source_url: &str) -> Vec { + let href = href.trim(); + let Some(root) = vault_root(current_source_url) else { + return vec![]; + }; + let dir = source_dir(current_source_url); + + let resolved = if href.starts_with("./") { + format!("{}{}", dir, &href[2..]) + } else if href.starts_with("../") { + resolve_parent(dir, &href[3..], root) + } else { + // Relative to vault root (Obsidian default for bare paths) + format!("{}{}", root, href) + }; + + candidates_with_and_without_extension(&resolved) +} + +fn resolve_parent(current_dir: &str, rest: &str, vault_root: &str) -> String { + let dir = current_dir.trim_end_matches('/'); + let parent = dir + .rfind('/') + .map(|p| &dir[..p + 1]) + .unwrap_or(vault_root); + if rest.starts_with("../") { + resolve_parent(parent, &rest[3..], vault_root) + } else { + format!("{}{}", parent, rest) + } +} + +/// For a wikilink `[[Name]]`, build exact-URL candidates to try first. +/// Wikilinks resolve by filename anywhere in the vault, so we generate: +/// - `vault_root/Name.md` +/// - `vault_root/name.md` (lower-case stem) +/// - `vault_root/name-with-dashes.md` (slug variant) +/// +/// The `find_atoms_by_wikilink_name` SQL fallback handles subdirectory +/// resolution when none of these exact hits land. +fn build_wikilink_exact_candidates(name: &str, current_source_url: &str) -> Vec { + let Some(root) = vault_root(current_source_url) else { + return vec![]; + }; + let slug = name.to_lowercase().replace(' ', "-"); + let mut candidates = vec![ + format!("{}{}.md", root, name), + format!("{}{}.md", root, name.to_lowercase()), + format!("{}{}.md", root, slug), + ]; + candidates.dedup(); + candidates +} + +/// Return the URL itself plus a variant without the `.md` extension (and +/// vice-versa), so callers can match atoms stored either way. +fn candidates_with_and_without_extension(url: &str) -> Vec { + if url.ends_with(".md") { + vec![url.to_string(), url[..url.len() - 3].to_string()] + } else { + vec![url.to_string(), format!("{}.md", url)] + } +} + +// ==================== Display-text extraction ==================== + +/// Scan backwards from the `]` at byte index `bracket_pos` to find the +/// matching `[`, returning `(full_original_text, display_text)`. +fn scan_back_for_display_text(content: &str, bracket_pos: usize) -> (String, String) { + let bytes = content.as_bytes(); + if bracket_pos == 0 { + return (String::new(), String::new()); + } + + // Walk backwards through the content to find the opening `[` + let mut depth = 1usize; + let mut k = bracket_pos.saturating_sub(1); + loop { + match bytes[k] { + b']' => depth += 1, + b'[' => { + depth -= 1; + if depth == 0 { + break; + } + } + _ => {} + } + if k == 0 { + break; + } + k -= 1; + } + + // Full match spans from `[` to the `)` that closes the href. + // The `)` position is not available here — callers use `original` only + // for replacement, so we capture the `[display]` part; the `(href)` part + // is appended by the caller when building `original`. + let display = std::str::from_utf8(&bytes[k + 1..bracket_pos]) + .unwrap_or("") + .to_string(); + + // We don't have the closing `)` index here, so return empty for `original` + // and let the caller reconstruct it. + (String::new(), display) +} + +// ==================== Replacement ==================== + +/// Apply resolved link replacements to `content`, returning the updated string. +/// +/// Each replacement: `(original_text, new_href)` — the display text is +/// preserved; only the href portion is changed. +pub fn apply_link_replacements(content: &str, replacements: &[ResolvedLink]) -> String { + let mut result = content.to_string(); + + for resolved in replacements { + // For markdown links: [text](old_href) → [text](atom://id) + // For wikilinks: [[Name]] → [Name](atom://id) + let original = &resolved.original; + let new_href = &resolved.replacement; + + if original.starts_with("[[") { + // Wikilink → markdown link with atom:// href + let inner = &original[2..original.len() - 2]; + let display = inner.split('|').next().unwrap_or(inner).trim(); + let replacement = format!("[{}]({})", display, new_href); + result = result.replacen(original.as_str(), &replacement, 1); + } else if let (Some(open), Some(close)) = (original.find("]("), original.rfind(')')) { + // Markdown link → update only the href part + let display = &original[1..open]; + let replacement = format!("[{}]({})", display, new_href); + result = result.replacen(original.as_str(), &replacement, 1); + } + } + + result +} + +/// Reconstruct the full original markdown link text `[display](href)` for a +/// given href and its position in `content`, so we can build `InternalLink.original`. +/// +/// Called after extraction to fill in the `original` field that +/// `scan_back_for_display_text` could not complete. +pub fn build_original_text(display: &str, href: &str) -> String { + format!("[{}]({})", display, href) +} + +// ==================== Tests ==================== + +#[cfg(test)] +mod tests { + use super::*; + + const VAULT: &str = "obsidian://ar-playbook/"; + const SOURCE: &str = "obsidian://ar-playbook/processes/deployment.md"; + + #[test] + fn test_relative_href_resolves_to_vault_root() { + let candidates = build_href_candidates("processes/work-tracking.md", SOURCE); + assert!(candidates.contains(&"obsidian://ar-playbook/processes/work-tracking.md".to_string())); + } + + #[test] + fn test_dotslash_href_resolves_relative_to_current_dir() { + let candidates = build_href_candidates("./capacity-planning.md", SOURCE); + assert!(candidates.contains( + &"obsidian://ar-playbook/processes/capacity-planning.md".to_string() + )); + } + + #[test] + fn test_parent_href_resolves_correctly() { + let candidates = build_href_candidates("../docs/overview.md", SOURCE); + assert!(candidates.contains(&"obsidian://ar-playbook/docs/overview.md".to_string())); + } + + #[test] + fn test_absolute_url_not_internal() { + assert!(!is_internal_href("https://example.com/file.md")); + assert!(!is_internal_href("http://example.com")); + assert!(!is_internal_href("obsidian://vault/path.md")); + assert!(!is_internal_href("atom://some-id")); + } + + #[test] + fn test_relative_path_is_internal() { + assert!(is_internal_href("processes/work-tracking.md")); + assert!(is_internal_href("./capacity.md")); + assert!(is_internal_href("../docs/overview.md")); + } + + #[test] + fn test_fragment_not_internal() { + assert!(!is_internal_href("#section-heading")); + assert!(!is_internal_href("")); + } + + #[test] + fn test_extract_markdown_links() { + let content = "See [Work Tracking](processes/work-tracking.md) and [Metrics](../docs/metrics.md)."; + let links = extract_internal_links(content, Some(SOURCE)); + assert_eq!(links.len(), 2); + let hrefs: Vec<&str> = links.iter().map(|l| l.href.as_str()).collect(); + assert!(hrefs.contains(&"processes/work-tracking.md")); + assert!(hrefs.contains(&"../docs/metrics.md")); + } + + #[test] + fn test_extract_wikilinks() { + let content = "See [[Work Tracking]] and [[Metrics|Metrics Docs]]."; + let links = extract_internal_links(content, Some(SOURCE)); + assert_eq!(links.len(), 2); + assert_eq!(links[0].href, "Work Tracking"); + assert_eq!(links[1].href, "Metrics"); + } + + #[test] + fn test_no_links_in_plain_text() { + let content = "No links here. Just text."; + let links = extract_internal_links(content, Some(SOURCE)); + assert!(links.is_empty()); + } + + #[test] + fn test_absolute_links_ignored() { + let content = "See [Confluence](https://atlassian.net/wiki/page) and [Source](obsidian://vault/file.md)."; + let links = extract_internal_links(content, Some(SOURCE)); + assert!(links.is_empty()); + } + + #[test] + fn test_vault_root_extraction() { + assert_eq!( + vault_root("obsidian://ar-playbook/processes/deployment.md"), + Some("obsidian://ar-playbook/") + ); + } + + #[test] + fn test_apply_markdown_replacement() { + let content = "See [Work Tracking](processes/work-tracking.md)."; + let resolved = vec![ResolvedLink { + original: "[Work Tracking](processes/work-tracking.md)".to_string(), + target_atom_id: "abc123".to_string(), + replacement: "atom://abc123".to_string(), + }]; + let result = apply_link_replacements(content, &resolved); + assert_eq!(result, "See [Work Tracking](atom://abc123)."); + } + + #[test] + fn test_apply_wikilink_replacement() { + let content = "See [[Work Tracking]] for details."; + let resolved = vec![ResolvedLink { + original: "[[Work Tracking]]".to_string(), + target_atom_id: "abc123".to_string(), + replacement: "atom://abc123".to_string(), + }]; + let result = apply_link_replacements(content, &resolved); + assert_eq!(result, "See [Work Tracking](atom://abc123) for details."); + } +} diff --git a/crates/atomic-core/src/health/llm_fixes.rs b/crates/atomic-core/src/health/llm_fixes.rs new file mode 100644 index 00000000..7e3604ad --- /dev/null +++ b/crates/atomic-core/src/health/llm_fixes.rs @@ -0,0 +1,226 @@ +//! LLM-powered fix implementations. +//! +//! These fixes call the configured LLM provider to make judgment calls that +//! deterministic SQL cannot. All are logged for undo. +//! +//! Currently implemented: +//! - `fix_untagged_atoms` — re-run tagging pipeline on zero-tag complete atoms. +//! - `merge_duplicate_pair` — synthesise two high-similarity atoms into one. + +use crate::error::AtomicCoreError; +use crate::health::{audit, FixAction}; +use crate::providers::{create_llm_provider, ProviderConfig}; + +use crate::providers::{LlmConfig}; +use crate::providers::types::Message; +use crate::AtomicCore; +use serde_json::json; + +/// Re-run the tagging pipeline on atoms that completed tagging but got 0 tags. +pub async fn fix_untagged_complete_atoms( + core: &AtomicCore, + untagged_ids: &[String], + dry_run: bool, +) -> Result, AtomicCoreError> { + if untagged_ids.is_empty() { + return Ok(None); + } + + let count = untagged_ids.len() as i32; + + let id = if dry_run { + "dry_run".to_string() + } else { + // Reset tagging status to pending so the pipeline picks them up + for atom_id in untagged_ids { + let _ = core + .storage() + .set_tagging_status_sync(atom_id, "pending", None) + .await; + } + + // Trigger the tagging pipeline + let processed = core.process_pending_tagging(|_| {}).await.unwrap_or(0); + tracing::info!(count, processed, "llm_fixes: re-queued untagged atoms for tagging"); + + audit::log_fix( + core, + "tagging_coverage", + "requeued_untagged_for_tagging", + "low", + Some(untagged_ids), + None, + json!({"atom_ids": untagged_ids}), + json!({"requeued": count, "processed": processed}), + None, + None, + ) + .await? + }; + + Ok(Some(FixAction { + id, + check: "tagging_coverage".to_string(), + action: "requeued_untagged_for_tagging".to_string(), + count, + details: untagged_ids.iter().take(10).cloned().collect(), + })) +} + +/// Merge two highly-similar atoms using the LLM. +/// +/// The LLM synthesises both atoms into one coherent document, then: +/// 1. Updates the newer atom with the merged content. +/// 2. Deletes the older atom. +/// 3. Re-queues the merged atom for embedding + tagging. +/// +/// This is a High-tier action and must be explicitly requested via +/// `POST /api/health/fix/{check}/{item_id}`. +pub async fn merge_duplicate_pair( + core: &AtomicCore, + atom_a_id: &str, + atom_b_id: &str, + dry_run: bool, +) -> Result, AtomicCoreError> { + let Some(atom_a) = core.get_atom(atom_a_id).await? else { + return Err(AtomicCoreError::NotFound(format!("atom {atom_a_id} not found"))); + }; + let Some(atom_b) = core.get_atom(atom_b_id).await? else { + return Err(AtomicCoreError::NotFound(format!("atom {atom_b_id} not found"))); + }; + + // Determine which is newer (keep) and which is older (delete) + let (keep, delete) = if atom_a.atom.updated_at >= atom_b.atom.updated_at { + (atom_a, atom_b) + } else { + (atom_b, atom_a) + }; + + let merge_prompt = format!( + "You are merging two duplicate knowledge base atoms into one definitive version.\n\n\ + ATOM A (source: {source_a}, created: {date_a}):\n{content_a}\n\n\ + ATOM B (source: {source_b}, created: {date_b}):\n{content_b}\n\n\ + Rules:\n\ + - Combine all unique information from both atoms into one coherent document\n\ + - If they contradict each other, prefer the more recent source\n\ + - Preserve all actionable details (URLs, commands, config values)\n\ + - Use clean markdown with proper headings\n\ + - Add a '## Sources' section at the bottom listing both original source URLs\n\ + - Do not add commentary — just produce the merged document\n\n\ + Output the merged markdown only.", + source_a = keep.atom.source_url.as_deref().unwrap_or("manual"), + date_a = keep.atom.created_at, + content_a = &keep.atom.content, + source_b = delete.atom.source_url.as_deref().unwrap_or("manual"), + date_b = delete.atom.created_at, + content_b = &delete.atom.content, + ); + + if dry_run { + return Ok(Some(FixAction { + id: "dry_run".to_string(), + check: "duplicate_detection".to_string(), + action: "merge_with_llm".to_string(), + count: 1, + details: vec![format!("Would merge {} into {}", delete.atom.id, keep.atom.id)], + })); + } + + // Get LLM provider + let settings = core.get_settings_map().await.unwrap_or_default(); + let provider_config = ProviderConfig::from_settings(&settings); + let llm = create_llm_provider(&provider_config).map_err(|e| { + AtomicCoreError::Configuration(format!("LLM provider unavailable for merge: {e}")) + })?; + + let model = settings + .get("wiki_model") + .cloned() + .unwrap_or_else(|| "anthropic/claude-sonnet-4.6".to_string()); + + let messages = vec![Message::user(merge_prompt.clone())]; + let config = LlmConfig::new(model).with_params( + crate::providers::types::GenerationParams::new().with_max_tokens(4096), + ); + + let response = llm.complete(&messages, &config).await?; + let merged_content = response.content.clone(); + + if merged_content.is_empty() { + return Err(AtomicCoreError::Validation( + "LLM returned empty merged content".to_string(), + )); + } + + // Capture before state + let before_state = json!([ + { + "id": keep.atom.id, + "content": keep.atom.content, + "source_url": keep.atom.source_url, + "tag_ids": keep.tags.iter().map(|t| t.id.clone()).collect::>() + }, + { + "id": delete.atom.id, + "content": delete.atom.content, + "source_url": delete.atom.source_url, + "tag_ids": delete.tags.iter().map(|t| t.id.clone()).collect::>() + } + ]); + + // Merge tags from deleted atom into keeper + let delete_tag_ids: Vec = delete.tags.iter().map(|t| t.id.clone()).collect(); + if !delete_tag_ids.is_empty() { + let _ = core + .storage() + .link_tags_to_atom_impl(&keep.atom.id, &delete_tag_ids) + .await; + } + + // Update the keeper with merged content + let upd = crate::UpdateAtomRequest { + content: merged_content.clone(), + source_url: keep.atom.source_url.clone(), + published_at: None, + tag_ids: None, + }; + core.update_atom(&keep.atom.id, upd, |_| {}).await?; + + // Delete the older atom + core.delete_atom(&delete.atom.id).await?; + + let fix_id = audit::log_fix( + core, + "duplicate_detection", + "merge_with_llm", + "high", + Some(&[keep.atom.id.clone(), delete.atom.id.clone()]), + None, + before_state, + json!({ + "kept_id": keep.atom.id, + "deleted_id": delete.atom.id, + "merged_content_length": merged_content.len() + }), + Some(&merge_prompt), + Some(&merged_content), + ) + .await?; + + tracing::info!( + kept = %keep.atom.id, + deleted = %delete.atom.id, + "duplicate pair merged with LLM" + ); + + Ok(Some(FixAction { + id: fix_id, + check: "duplicate_detection".to_string(), + action: "merge_with_llm".to_string(), + count: 1, + details: vec![ + format!("Kept: {}", keep.atom.id), + format!("Deleted: {}", delete.atom.id), + ], + })) +} diff --git a/crates/atomic-core/src/health/mod.rs b/crates/atomic-core/src/health/mod.rs new file mode 100644 index 00000000..8735f841 --- /dev/null +++ b/crates/atomic-core/src/health/mod.rs @@ -0,0 +1,658 @@ +//! Knowledge-base health monitoring and auto-remediation. +//! +//! This module computes a scored health report across 10 checks, each targeting +//! a distinct class of data-quality issue. Deterministic fixes (orphan-tag +//! deletion, retry pipelines, graph rebuild) run automatically at "safe" or +//! "low" tier. LLM-powered fixes (merge duplicates, enrich stubs, structure +//! content) are available but always logged and undoable. +//! +//! # Flow +//! 1. `compute_health(core)` → runs all checks → returns `HealthReport` +//! 2. `run_fix(core, req)` → applies fixes by tier → returns `FixResponse` +//! 3. `undo_fix(core, fix_id)` → restores pre-fix state from audit log +//! +//! # Check weights (must sum to 1.0) +//! | Check | Weight | +//! |---------------------------|--------| +//! | duplicate_detection | 15 % | +//! | embedding_coverage | 15 % | +//! | tagging_coverage | 20 % | +//! | source_uniqueness | 10 % | +//! | wiki_coverage | 10 % | +//! | semantic_graph_freshness | 10 % | +//! | content_quality | 5 % | +//! | orphan_tags | 5 % | +//! | tag_health | 5 % | +//! | contradiction_detection | 5 % | + +pub mod audit; +pub mod checks; +pub mod fixes; +pub mod link_resolution; +pub mod llm_fixes; +pub mod task; + +use crate::error::AtomicCoreError; +use crate::AtomicCore; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +// ==================== Core types ==================== + +/// Overall status derived from the numeric score. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum HealthStatus { + Healthy, + NeedsAttention, + Degraded, + Unhealthy, +} + +impl HealthStatus { + pub fn from_score(score: u32) -> Self { + match score { + 90..=100 => Self::Healthy, + 70..=89 => Self::NeedsAttention, + 50..=69 => Self::Degraded, + _ => Self::Unhealthy, + } + } + pub fn as_str(&self) -> &'static str { + match self { + Self::Healthy => "healthy", + Self::NeedsAttention => "needs_attention", + Self::Degraded => "degraded", + Self::Unhealthy => "unhealthy", + } + } +} + +/// Result for one individual health check. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthCheckResult { + /// "ok" | "warning" | "error" + pub status: String, + /// 0–100 contribution to the overall score + pub score: u32, + pub auto_fixable: bool, + pub requires_review: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub fix_action: Option, + /// Check-specific numbers, lists, pairs, etc. + pub data: serde_json::Value, +} + +/// Complete health report returned by `GET /api/health/knowledge`. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthReport { + pub overall_score: u32, + pub overall_status: String, + pub computed_at: String, + pub atom_count: i32, + pub checks: HashMap, + pub auto_fixable: i32, + pub requires_review: i32, + #[serde(skip_serializing_if = "Option::is_none")] + pub previous_score: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub previous_check_scores: Option>, +} + +/// A single action taken (or that would be taken) during a fix run. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FixAction { + /// ID of the `health_fix_log` row (for undo). + pub id: String, + pub check: String, + pub action: String, + pub count: i32, + pub details: Vec, +} + +/// An issue that was skipped (too high tier, or no-op). +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SkippedFix { + pub check: String, + pub reason: String, + pub count: i32, +} + +/// Response from `POST /api/health/fix`. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FixResponse { + pub mode: String, + pub actions_taken: Vec, + pub skipped: Vec, + pub new_score: u32, +} + +/// Fix safety tier. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FixTier { + /// Retry pipelines, process pending — zero risk. + Safe, + /// Delete orphan tags, generate missing wikis — logged, undoable. + Low, + /// Modify content (add headings, merge exact-source dupes) — dry-run first. + Medium, + /// Merges, splits, deletes — always requires user confirmation. + High, +} + +impl FixTier { + pub fn as_str(&self) -> &'static str { + match self { + Self::Safe => "safe", + Self::Low => "low", + Self::Medium => "medium", + Self::High => "high", + } + } + pub fn from_str(s: &str) -> Self { + match s { + "low" => Self::Low, + "medium" => Self::Medium, + "high" => Self::High, + _ => Self::Safe, + } + } +} + +/// What the caller wants the fix run to do. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FixRequest { + /// Which checks to fix; `None` = all auto-fixable checks. + pub checks: Option>, + /// "auto" = execute changes; "dry_run" = report without executing. + pub mode: String, + /// Include Medium-tier fixes (default false). + #[serde(default)] + pub include_medium: bool, +} + +impl FixRequest { + pub fn is_dry_run(&self) -> bool { + self.mode == "dry_run" + } + pub fn max_tier(&self) -> FixTier { + if self.include_medium { + FixTier::Medium + } else { + FixTier::Low + } + } +} + +// ==================== Raw data types used across checks ==================== + +/// Atom pair with high similarity (potential duplicate). +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DuplicatePair { + pub pair_id: String, + pub atom_a_id: String, + pub atom_a_title: String, + pub atom_a_source: Option, + pub atom_b_id: String, + pub atom_b_title: String, + pub atom_b_source: Option, + pub similarity: f32, + /// Number of tags shared between the two atoms (higher = more likely related). + pub shared_tag_count: i32, +} + +/// Tag eligible for wiki that doesn't have one yet. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WikiGap { + pub tag_id: String, + pub tag_name: String, + pub atom_count: i32, +} + +/// Wiki that exists but is out of date. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WikiStaleEntry { + pub tag_id: String, + pub tag_name: String, + pub new_atom_count: i32, +} + +// ==================== Orchestrator ==================== + +/// Check weights. Must sum to 1.0. +const CHECK_WEIGHTS: &[(&str, f64)] = &[ + ("content_overlap", 0.15), + ("embedding_coverage", 0.15), + ("tagging_coverage", 0.20), + ("source_uniqueness", 0.10), + ("wiki_coverage", 0.10), + ("semantic_graph_freshness", 0.10), + ("content_quality", 0.05), + ("orphan_tags", 0.05), + ("tag_health", 0.05), + ("broken_internal_links", 0.05), +]; + +/// Run all health checks and return a complete `HealthReport`. +/// +/// Completes in < 2s for databases with up to ~1,000 atoms. Contradiction +/// detection is a stub (no LLM call) so it won't time out on large graphs. +pub async fn compute_health(core: &AtomicCore) -> Result { + let computed_at = chrono::Utc::now().to_rfc3339(); + + // Fetch all raw data in a single spawn_blocking pass + let raw = core.storage().health_check_data_sync().await?; + + // Run all synchronous checks + let mut checks = checks::run_all(&raw); + + // Run async link-resolution check (needs DB lookups per candidate atom) + match compute_link_check(core).await { + Ok(link_check) => { + checks.insert("broken_internal_links".to_string(), link_check); + } + Err(e) => { + tracing::warn!(error = %e, "broken_internal_links check failed"); + } + } + + // Aggregate score + let overall_score = aggregate_score(&checks); + let overall_status = HealthStatus::from_score(overall_score).as_str().to_string(); + + // Count auto-fixable vs requires-review + let auto_fixable = checks + .values() + .filter(|c| c.auto_fixable && c.status != "ok") + .count() as i32; + let requires_review = checks + .values() + .filter(|c| c.requires_review && c.status != "ok") + .count() as i32; + + let atom_count = raw.total_atoms; + + // Fetch previous report for trending (before storing the current one) + let (previous_score, previous_check_scores) = + match core.get_latest_health_report().await { + Ok(Some(prev)) => { + let check_scores: HashMap = + prev.checks.iter().map(|(k, v)| (k.clone(), v.score)).collect(); + (Some(prev.overall_score), Some(check_scores)) + } + _ => (None, None), + }; + + let report = HealthReport { + overall_score, + overall_status, + computed_at: computed_at.clone(), + atom_count, + checks, + auto_fixable, + requires_review, + previous_score, + previous_check_scores, + }; + + // Persist for trending (fire-and-forget; ignore errors) + let _ = store_report(core, &report).await; + + Ok(report) +} + +/// Compute a single named health check in isolation. +/// +/// Accepts any check name from the standard set. For the async +/// `broken_internal_links` check, runs `compute_link_check` directly. +/// Returns `(check_name, HealthCheckResult)` so callers can update +/// a cached `HealthReport` in place. +pub async fn compute_single_check( + core: &AtomicCore, + check_name: &str, +) -> Result<(String, HealthCheckResult), AtomicCoreError> { + let result = match check_name { + // Async check — requires per-atom DB lookups + "broken_internal_links" => compute_link_check(core).await?, + // Sync checks — fetch raw data once, dispatch to the appropriate fn + "embedding_coverage" + | "tagging_coverage" + | "content_overlap" + | "source_uniqueness" + | "wiki_coverage" + | "semantic_graph_freshness" + | "content_quality" + | "orphan_tags" + | "tag_health" + | "contradiction_detection" + | "boilerplate_pollution" => { + let raw = core.storage().health_check_data_sync().await?; + match check_name { + "embedding_coverage" => checks::embedding_coverage(&raw), + "tagging_coverage" => checks::tagging_coverage(&raw), + "content_overlap" => checks::content_overlap(&raw), + "source_uniqueness" => checks::source_uniqueness(&raw), + "wiki_coverage" => checks::wiki_coverage(&raw), + "semantic_graph_freshness" => checks::semantic_graph_freshness(&raw), + "content_quality" => checks::content_quality(&raw), + "orphan_tags" => checks::orphan_tags(&raw), + "tag_health" => checks::tag_health(&raw), + "contradiction_detection" => checks::contradiction_detection(&raw), + "boilerplate_pollution" => checks::boilerplate_pollution(&raw), + _ => unreachable!(), + } + } + _ => { + return Err(AtomicCoreError::Validation(format!( + "Unknown health check: {check_name}" + ))) + } + }; + Ok((check_name.to_string(), result)) +} + +/// Store a completed report in the health_reports table. +async fn store_report( + core: &AtomicCore, + report: &HealthReport, +) -> Result<(), AtomicCoreError> { + use crate::health::audit::StoredHealthReport; + let check_scores: HashMap = report + .checks + .iter() + .map(|(k, v)| (k.clone(), v.score)) + .collect(); + let stored = StoredHealthReport { + id: uuid::Uuid::new_v4().to_string(), + computed_at: report.computed_at.clone(), + overall_score: report.overall_score, + check_scores: serde_json::to_string(&check_scores).unwrap_or_default(), + atom_count: report.atom_count, + auto_fixes_applied: 0, + report_json: serde_json::to_string(report).unwrap_or_default(), + }; + core.storage().store_health_report_sync(&stored).await +} + +/// Async health check for broken internal links. +/// +/// Fetches candidate atoms, extracts internal links, resolves each to a +/// target source URL, and counts how many could not be resolved to an +/// existing atom. Runs outside `health_check_data_sync` because it needs +/// multiple async DB round-trips. +async fn compute_link_check(core: &AtomicCore) -> Result { + use link_resolution::{extract_internal_links, vault_root}; + + let candidates = core.storage().get_link_candidate_atoms_sync().await?; + if candidates.is_empty() { + return Ok(HealthCheckResult { + status: "ok".to_string(), + score: 100, + auto_fixable: false, + requires_review: false, + fix_action: None, + data: serde_json::json!({ "broken_count": 0, "affected_atoms": 0 }), + }); + } + + let mut broken_count = 0i32; + let mut affected_atoms = 0i32; + + for (atom_id, content, source_url) in &candidates { + let links = extract_internal_links(content, source_url.as_deref()); + if links.is_empty() { + continue; + } + + // Collect all candidate source URLs from this atom's links + let candidate_urls: Vec = links + .iter() + .flat_map(|l| l.candidate_source_urls.iter().cloned()) + .collect(); + + let url_map = core + .storage() + .find_atoms_by_source_urls_sync(candidate_urls) + .await + .unwrap_or_default(); + + let vault_pfx = source_url + .as_deref() + .and_then(vault_root) + .map(|s| s.to_string()); + + let mut atom_broken = false; + for link in &links { + // Resolved if any candidate URL matched + let resolved_by_url = link + .candidate_source_urls + .iter() + .any(|u| url_map.contains_key(u)); + + if resolved_by_url { + continue; + } + + // For wikilinks, also try the vault-wide name lookup + let resolved_by_name = if let (Some(name), Some(pfx)) = (&link.wikilink_name, &vault_pfx) { + core.storage() + .find_atom_by_wikilink_name_sync(name.clone(), pfx.clone()) + .await + .unwrap_or(None) + .is_some() + } else { + false + }; + + if !resolved_by_name { + broken_count += 1; + atom_broken = true; + } + } + if atom_broken { + affected_atoms += 1; + } + } + + // Score: proportion of all atoms that are clean of broken links. + // Fetch total atom count so a few broken atoms in a large KB + // don't collapse the score. + let total_atoms = core + .count_atoms() + .await + .unwrap_or(candidates.len() as i32); + let clean_atoms = (total_atoms - affected_atoms).max(0); + let score = if total_atoms == 0 { + 100 + } else { + (clean_atoms as f64 / total_atoms as f64 * 100.0) as u32 + }; + let status = if broken_count == 0 { "ok" } else { "warning" }; + + Ok(HealthCheckResult { + status: status.to_string(), + score, + auto_fixable: broken_count > 0, + requires_review: false, + fix_action: Some("resolve_internal_links".to_string()), + data: serde_json::json!({ + "broken_count": broken_count, + "affected_atoms": affected_atoms, + }), + }) +} + +/// Weighted average of all check scores. +pub fn aggregate_score(checks: &HashMap) -> u32 { + let mut total = 0.0_f64; + let mut weight_sum = 0.0_f64; + for (name, weight) in CHECK_WEIGHTS { + if let Some(check) = checks.get(*name) { + total += (check.score as f64) * weight; + weight_sum += weight; + } + } + if weight_sum == 0.0 { + return 100; + } + ((total / weight_sum).round() as u32).min(100) +} + +/// Run auto-fixes up to the requested tier. +pub async fn run_fix( + core: &AtomicCore, + req: &FixRequest, +) -> Result { + let raw = core.storage().health_check_data_sync().await?; + let checks = checks::run_all(&raw); + let max_tier = req.max_tier(); + let dry_run = req.is_dry_run(); + + let mut actions_taken: Vec = Vec::new(); + let mut skipped: Vec = Vec::new(); + + // Helper: should we run this check's fix? + let should_run = |check_name: &str| -> bool { + if let Some(filter) = &req.checks { + filter.iter().any(|c| c == check_name) + } else { + true + } + }; + + // --- Safe tier --- + + if should_run("embedding_coverage") { + if let Some(check) = checks.get("embedding_coverage") { + if check.auto_fixable && check.status != "ok" { + match fixes::fix_embedding_coverage(core, dry_run).await { + Ok(Some(action)) => actions_taken.push(action), + Ok(None) => {} + Err(e) => { + tracing::warn!(error = %e, "embedding_coverage fix failed"); + } + } + } + } + } + + if should_run("semantic_graph_freshness") { + if let Some(check) = checks.get("semantic_graph_freshness") { + if check.auto_fixable && check.status != "ok" { + match fixes::fix_graph_freshness(core, dry_run).await { + Ok(Some(action)) => actions_taken.push(action), + Ok(None) => {} + Err(e) => { + tracing::warn!(error = %e, "semantic_graph_freshness fix failed"); + } + } + } + } + } + + if should_run("tagging_coverage") { + if let Some(check) = checks.get("tagging_coverage") { + if check.auto_fixable && check.status != "ok" { + let skipped_untagged = raw.skipped_untagged; + match fixes::fix_tagging_coverage(core, skipped_untagged, dry_run).await { + Ok(Some(action)) => actions_taken.push(action), + Ok(None) => {} + Err(e) => { + tracing::warn!(error = %e, "tagging_coverage fix failed"); + } + } + } + } + } + + // --- Low tier --- + + if matches!(max_tier, FixTier::Low | FixTier::Medium | FixTier::High) { + if should_run("orphan_tags") { + if let Some(check) = checks.get("orphan_tags") { + if check.auto_fixable && check.status != "ok" { + match fixes::fix_orphan_tags(core, &raw, dry_run).await { + Ok(Some(action)) => actions_taken.push(action), + Ok(None) => {} + Err(e) => tracing::warn!(error = %e, "orphan_tags fix failed"), + } + } + } + } + + if should_run("wiki_coverage") { + if let Some(check) = checks.get("wiki_coverage") { + if check.auto_fixable && check.status != "ok" { + match fixes::fix_wiki_coverage(core, &raw, dry_run).await { + Ok(Some(action)) => actions_taken.push(action), + Ok(None) => {} + Err(e) => tracing::warn!(error = %e, "wiki_coverage fix failed"), + } + } + } + } + } + + // --- Medium tier --- + + if matches!(max_tier, FixTier::Medium | FixTier::High) { + if should_run("source_uniqueness") { + if let Some(check) = checks.get("source_uniqueness") { + if check.auto_fixable && check.status != "ok" { + match fixes::fix_source_uniqueness(core, &raw, dry_run).await { + Ok(Some(action)) => actions_taken.push(action), + Ok(None) => {} + Err(e) => tracing::warn!(error = %e, "source_uniqueness fix failed"), + } + } + } + } + } + + if matches!(max_tier, FixTier::Medium | FixTier::High) && should_run("broken_internal_links") { + if matches!(checks.get("broken_internal_links"), Some(c) if c.auto_fixable && c.status != "ok") { + match fixes::fix_broken_internal_links(core, dry_run).await { + Ok(Some(action)) => actions_taken.push(action), + Ok(None) => tracing::debug!("broken_internal_links: no links to fix"), + Err(e) => tracing::warn!(error = %e, "broken_internal_links fix failed"), + } + } + } + // Mark high-tier issues as skipped with reason + for (check_name, check) in &checks { + if check.requires_review && check.status != "ok" && !should_run(check_name) { + skipped.push(SkippedFix { + check: check_name.clone(), + reason: "requires_review".to_string(), + count: check.data.get("count").and_then(|v| v.as_i64()).unwrap_or(0) + as i32, + }); + } + } + + // Recompute score after fixes (if not dry run) + let new_score = if !dry_run && !actions_taken.is_empty() { + let new_raw = core.storage().health_check_data_sync().await?; + let new_checks = checks::run_all(&new_raw); + aggregate_score(&new_checks) + } else { + aggregate_score(&checks) + }; + + Ok(FixResponse { + mode: req.mode.clone(), + actions_taken, + skipped, + new_score, + }) +} diff --git a/crates/atomic-core/src/health/task.rs b/crates/atomic-core/src/health/task.rs new file mode 100644 index 00000000..aa6623d7 --- /dev/null +++ b/crates/atomic-core/src/health/task.rs @@ -0,0 +1,107 @@ +//! Nightly health maintenance scheduled task. +//! +//! Runs daily at ~3 AM (configurable). Automatically applies Safe + Low tier +//! fixes and records the health report for trending. If the score drops below +//! 70, the next briefing run will include a health summary. + +use crate::health::{self, FixRequest}; +use crate::scheduler::{state as task_state, ScheduledTask, TaskContext, TaskError, TaskEvent}; +use crate::AtomicCore; +use async_trait::async_trait; +use std::time::Duration; + +pub struct HealthMaintenanceTask; + +const TASK_ID: &str = "health_maintenance"; +const DEFAULT_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60); +const DEFAULT_ENABLED: bool = true; + +#[async_trait] +impl ScheduledTask for HealthMaintenanceTask { + fn id(&self) -> &'static str { + TASK_ID + } + + fn display_name(&self) -> &'static str { + "Knowledge health maintenance" + } + + fn default_interval(&self) -> Duration { + DEFAULT_INTERVAL + } + + async fn run(&self, core: &AtomicCore, ctx: &TaskContext) -> Result<(), TaskError> { + if !task_state::is_enabled(core, TASK_ID, DEFAULT_ENABLED).await { + return Err(TaskError::Disabled); + } + if !task_state::is_due(core, TASK_ID, DEFAULT_INTERVAL, DEFAULT_ENABLED).await { + return Err(TaskError::NotDue); + } + + let db_id = core + .db_path() + .file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.to_string()) + .unwrap_or_else(|| "default".to_string()); + + (ctx.event_cb)(TaskEvent::Started { + task_id: TASK_ID.to_string(), + db_id: db_id.clone(), + }); + + // Run health check + let report = match health::compute_health(core).await { + Ok(r) => r, + Err(e) => { + let msg = e.to_string(); + (ctx.event_cb)(TaskEvent::Failed { + task_id: TASK_ID.to_string(), + db_id, + error: msg.clone(), + }); + return Err(TaskError::Other(msg)); + } + }; + + let score_before = report.overall_score; + tracing::info!( + score = score_before, + status = %report.overall_status, + "[health_maintenance] initial score" + ); + + // Auto-fix Safe + Low tier issues + let fix_req = FixRequest { + checks: None, + mode: "auto".to_string(), + include_medium: false, + }; + + match health::run_fix(core, &fix_req).await { + Ok(fix_resp) => { + tracing::info!( + fixes = fix_resp.actions_taken.len(), + new_score = fix_resp.new_score, + "[health_maintenance] fixes applied" + ); + } + Err(e) => { + tracing::warn!(error = %e, "[health_maintenance] fix run failed"); + } + } + + // Persist last_run + task_state::set_last_run(core, TASK_ID, chrono::Utc::now()) + .await + .ok(); + + (ctx.event_cb)(TaskEvent::Completed { + task_id: TASK_ID.to_string(), + db_id, + result_id: None, + }); + + Ok(()) + } +} diff --git a/crates/atomic-core/src/lib.rs b/crates/atomic-core/src/lib.rs index 703282c3..2d347359 100644 --- a/crates/atomic-core/src/lib.rs +++ b/crates/atomic-core/src/lib.rs @@ -42,6 +42,7 @@ pub mod executor; pub mod export; pub mod extraction; pub mod graph_maintenance; +pub mod health; pub mod import; pub mod ingest; pub mod manager; @@ -3571,6 +3572,50 @@ impl AtomicCore { pub async fn recompute_all_tag_embeddings(&self) -> Result { self.storage.recompute_all_tag_embeddings_sync().await } + + // ==================== Health ==================== + + /// Compute a full health report across all 10 checks. + pub async fn compute_health(&self) -> Result { + crate::health::compute_health(self).await + } + + /// Run auto-fixes up to the requested tier. Returns a `FixResponse` with + /// actions taken, skipped issues, and the new score. + pub async fn run_health_fix( + &self, + req: &crate::health::FixRequest, + ) -> Result { + crate::health::run_fix(self, req).await + } + + /// Undo a previously applied fix by its log ID. + pub async fn undo_health_fix(&self, fix_id: &str) -> Result<(), AtomicCoreError> { + crate::health::audit::undo(self, fix_id).await + } + + /// Fetch the most recently stored health report without recomputing. + pub async fn get_latest_health_report( + &self, + ) -> Result, AtomicCoreError> { + self.storage.get_latest_health_report_sync().await + } + + /// Fetch recent stored health reports for trend display. + pub async fn get_health_reports( + &self, + limit: i32, + ) -> Result, AtomicCoreError> { + self.storage.get_health_reports_sync(limit).await + } + + /// Fetch recent fix log entries (most recent first). + pub async fn get_recent_health_fixes( + &self, + limit: i32, + ) -> Result, AtomicCoreError> { + self.storage.get_recent_fixes_sync(limit).await + } } fn oauth_unavailable() -> AtomicCoreError { diff --git a/crates/atomic-core/src/models.rs b/crates/atomic-core/src/models.rs index 5be0473b..73c947c1 100644 --- a/crates/atomic-core/src/models.rs +++ b/crates/atomic-core/src/models.rs @@ -367,6 +367,10 @@ pub struct WikiArticleSummary { pub updated_at: String, pub atom_count: i32, pub inbound_links: i32, + /// Live count of atoms tagged under this tag hierarchy that have been added + /// since the article was last generated. Computed server-side via the same + /// recursive CTE used by `GET /api/wiki/{tag_id}/status`; never stale. + pub new_atoms_available: i32, } /// Inter-article wiki link (cross-reference between wiki articles) diff --git a/crates/atomic-core/src/settings.rs b/crates/atomic-core/src/settings.rs index b9ba5709..03f88af3 100644 --- a/crates/atomic-core/src/settings.rs +++ b/crates/atomic-core/src/settings.rs @@ -82,6 +82,15 @@ pub const DEFAULT_SETTINGS: &[(&str, &str)] = &[ ("task.draft_pipeline.enabled", "true"), ("task.draft_pipeline.interval_minutes", "1"), ("task.draft_pipeline.quiet_minutes", "1"), + // Health maintenance task + ("task.health_maintenance.enabled", "true"), + ("task.health_maintenance.interval_hours", "24"), + // Health LLM prompt templates (empty = use built-in defaults) + ("health.merge_duplicates_prompt", ""), + ("health.contradiction_detection_prompt", ""), + ("health.split_long_atom_prompt", ""), + ("health.enrich_stub_atom_prompt", ""), + ("health.add_structure_prompt", ""), ]; /// Migrate settings - add any missing default settings diff --git a/crates/atomic-core/src/storage/mod.rs b/crates/atomic-core/src/storage/mod.rs index 9c33367f..7e76af9e 100644 --- a/crates/atomic-core/src/storage/mod.rs +++ b/crates/atomic-core/src/storage/mod.rs @@ -130,6 +130,210 @@ impl StorageBackend { } } +impl StorageBackend { + // ==================== Health dispatch methods ==================== + // These are not part of the Storage trait — health is an internal concern. + // Postgres returns an error for all health operations (not yet supported). + + pub(crate) async fn health_check_data_sync( + &self, + ) -> Result { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + tokio::task::spawn_blocking(move || s.health_check_data_impl()) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Err(AtomicCoreError::DatabaseOperation( + "health checks not yet supported on Postgres storage".to_string(), + )), + } + } + + pub(crate) async fn store_health_report_sync( + &self, + report: &crate::health::audit::StoredHealthReport, + ) -> Result<(), AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + let report = report.clone(); + tokio::task::spawn_blocking(move || s.store_health_report_impl(&report)) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(()), + } + } + + pub(crate) async fn get_latest_health_report_sync( + &self, + ) -> Result, AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + tokio::task::spawn_blocking(move || s.get_latest_health_report_impl()) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(None), + } + } + + pub(crate) async fn get_health_reports_sync( + &self, + limit: i32, + ) -> Result, AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + tokio::task::spawn_blocking(move || s.get_health_reports_impl(limit)) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(vec![]), + } + } + + pub(crate) async fn log_fix_action_sync( + &self, + log: &crate::health::audit::HealthFixLog, + ) -> Result<(), AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + let log = log.clone(); + tokio::task::spawn_blocking(move || s.log_fix_action_impl(&log)) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(()), + } + } + + pub(crate) async fn get_fix_log_sync( + &self, + fix_id: &str, + ) -> Result, AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + let fix_id = fix_id.to_string(); + tokio::task::spawn_blocking(move || s.get_fix_log_impl(&fix_id)) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(None), + } + } + + pub(crate) async fn get_recent_fixes_sync( + &self, + limit: i32, + ) -> Result, AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + tokio::task::spawn_blocking(move || s.get_recent_fixes_impl(limit)) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(vec![]), + } + } + + pub(crate) async fn mark_fix_undone_sync(&self, fix_id: &str) -> Result<(), AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + let fix_id = fix_id.to_string(); + tokio::task::spawn_blocking(move || s.mark_fix_undone_impl(&fix_id)) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(()), + } + } + + pub(crate) async fn reset_skipped_untagged_to_pending_sync( + &self, + ) -> Result { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + tokio::task::spawn_blocking(move || { + s.reset_skipped_untagged_to_pending_impl() + }) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(0), + } + } + + // ==================== Link resolution dispatch ==================== + + pub(crate) async fn get_link_candidate_atoms_sync( + &self, + ) -> Result)>, AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + tokio::task::spawn_blocking(move || s.get_link_candidate_atoms_impl()) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(vec![]), + } + } + + pub(crate) async fn find_atoms_by_source_urls_sync( + &self, + urls: Vec, + ) -> Result, AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + tokio::task::spawn_blocking(move || s.find_atoms_by_source_urls_impl(&urls)) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(std::collections::HashMap::new()), + } + } + + pub(crate) async fn find_atom_by_wikilink_name_sync( + &self, + name: String, + vault_prefix: String, + ) -> Result, AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + tokio::task::spawn_blocking(move || { + s.find_atom_by_wikilink_name_impl(&name, &vault_prefix) + }) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(None), + } + } +} + // ==================== Async dispatch methods ==================== // // Each method dispatches to either the SqliteStorage sync helper diff --git a/crates/atomic-core/src/storage/postgres/wiki.rs b/crates/atomic-core/src/storage/postgres/wiki.rs index 8711a793..8307d013 100644 --- a/crates/atomic-core/src/storage/postgres/wiki.rs +++ b/crates/atomic-core/src/storage/postgres/wiki.rs @@ -374,11 +374,42 @@ impl WikiStore for PostgresStorage { } async fn get_all_wiki_articles(&self) -> StorageResult> { - let rows = sqlx::query_as::<_, (String, String, String, String, i32, i64)>( - "SELECT w.id, w.tag_id, t.name, w.updated_at, w.atom_count, - (SELECT COUNT(*) FROM wiki_links wl WHERE wl.target_tag_id = w.tag_id AND wl.db_id = $1) + // Use a recursive CTE to compute the live atom count per tag hierarchy so that + // new_atoms_available is always consistent with GET /api/wiki/{tag_id}/status. + let rows = sqlx::query_as::<_, (String, String, String, String, i32, i64, i32)>( + "WITH RECURSIVE + -- Expand each wiki-article tag to include all its descendant tags. + -- Seeded only from tags that have a wiki article so the recursion is + -- bounded by the number of articles, not the full tag tree. + tag_tree(root_id, id) AS ( + SELECT t.id, t.id + FROM tags t + WHERE t.db_id = $1 + AND EXISTS (SELECT 1 FROM wiki_articles wa WHERE wa.tag_id = t.id AND wa.db_id = $1) + UNION ALL + SELECT tt.root_id, t.id + FROM tags t + JOIN tag_tree tt ON t.parent_id = tt.id + WHERE t.db_id = $1 + ), + -- Live atom count per root tag (counts atoms in the entire subtree). + live_counts(tag_id, cnt) AS ( + SELECT tt.root_id, COUNT(DISTINCT at.atom_id)::int + FROM tag_tree tt + JOIN atom_tags at ON at.tag_id = tt.id AND at.db_id = $1 + GROUP BY tt.root_id + ) + SELECT + w.id, + w.tag_id, + t.name, + w.updated_at, + w.atom_count, + (SELECT COUNT(*) FROM wiki_links wl WHERE wl.target_tag_id = w.tag_id AND wl.db_id = $1), + GREATEST(0, COALESCE(lc.cnt, 0) - w.atom_count) FROM wiki_articles w JOIN tags t ON w.tag_id = t.id AND t.db_id = $1 + LEFT JOIN live_counts lc ON lc.tag_id = w.tag_id WHERE w.db_id = $1 ORDER BY (SELECT COUNT(*) FROM wiki_links wl WHERE wl.target_tag_id = w.tag_id AND wl.db_id = $1) DESC, w.atom_count DESC, w.updated_at DESC", @@ -391,7 +422,7 @@ impl WikiStore for PostgresStorage { Ok(rows .into_iter() .map( - |(id, tag_id, tag_name, updated_at, atom_count, inbound_links)| { + |(id, tag_id, tag_name, updated_at, atom_count, inbound_links, new_atoms_available)| { WikiArticleSummary { id, tag_id, @@ -399,6 +430,7 @@ impl WikiStore for PostgresStorage { updated_at, atom_count, inbound_links: inbound_links as i32, + new_atoms_available, } }, ) diff --git a/crates/atomic-core/src/storage/sqlite/health.rs b/crates/atomic-core/src/storage/sqlite/health.rs new file mode 100644 index 00000000..2e96c706 --- /dev/null +++ b/crates/atomic-core/src/storage/sqlite/health.rs @@ -0,0 +1,798 @@ +//! SQLite-backed storage for health check raw data and the two health tables +//! (`health_reports`, `health_fix_log`). +//! +//! All methods here are synchronous (run inside `tokio::task::spawn_blocking`). + +use crate::error::AtomicCoreError; +use crate::health::audit::{HealthFixLog, StoredHealthReport}; +use crate::health::{DuplicatePair, WikiGap, WikiStaleEntry}; +use crate::storage::sqlite::SqliteStorage; +use rusqlite::params; + +// ==================== Raw health data ==================== + +/// All data needed by the health checks, fetched in a single blocking pass. +#[derive(Debug, Clone, Default)] +pub struct HealthRawData { + // — totals — + pub total_atoms: i32, + + // — embedding coverage — + pub embedding_pending: i32, + pub embedding_processing: i32, + pub embedding_complete: i32, + pub embedding_failed: i32, + + // — tagging coverage — + pub tagging_pending: i32, + pub tagging_processing: i32, + pub tagging_complete: i32, + pub tagging_failed: i32, + pub tagging_skipped: i32, + /// Atoms whose tagging_status = 'complete' but have 0 tags assigned. + pub untagged_complete: i32, + /// Atoms whose tagging_status = 'skipped' AND have 0 tags (invisible gap). + pub skipped_untagged: i32, + + // — source uniqueness — + /// `(source_url, [atom_id, ...])` for URLs that appear > 1 time. + pub duplicate_sources: Vec<(String, Vec)>, + + // — orphan tags — + /// `(id, name)` for tags with 0 atoms and no children (excluding autotag targets). + pub orphan_tags: Vec<(String, String)>, + + // — semantic graph freshness — + pub newest_atom_updated_at: Option, + pub newest_edge_created_at: Option, + /// Count of atoms whose `updated_at` > `newest_edge_created_at`. + pub atoms_since_edge_rebuild: i32, + + // — wiki coverage — + pub wiki_eligible_count: i32, + pub wiki_present_count: i32, + pub wiki_stale_count: i32, + pub wiki_gaps: Vec, + pub wiki_stale: Vec, + + // — content quality — + /// Atom IDs with content length < 100 chars. + pub very_short_atoms: Vec, + /// Atom IDs with content length > 15 000 chars. + pub very_long_atoms: Vec, + /// Atom IDs with no markdown heading (`#` at start of line). + pub no_heading_atoms: Vec, + /// Atom IDs with null source_url and no "Source:" text in content. + pub no_source_atoms: Vec, + + // — tag health — + pub single_atom_tags: i32, + pub rootless_tags: i32, + pub similar_name_pair_count: i32, + + // — duplicate detection (similarity >= 0.92) — + pub duplicate_pairs: Vec, + + // — boilerplate pollution (atoms with >= 2 edges at similarity >= 0.99) — + /// Atom IDs whose embeddings are dominated by shared template text. + pub boilerplate_affected_atoms: Vec, + + // — contradiction candidates (similarity 0.75..0.92) — + pub contradiction_pairs_checked: i32, + pub contradiction_candidate_count: i32, +} + +impl SqliteStorage { + /// Gather all raw health-check data in a single blocking pass. + pub(crate) fn health_check_data_impl(&self) -> Result { + let conn = self.db.read_conn()?; + let mut raw = HealthRawData::default(); + + // ---- total atoms ---- + raw.total_atoms = conn.query_row("SELECT COUNT(*) FROM atoms", [], |r| r.get(0))?; + + if raw.total_atoms == 0 { + return Ok(raw); + } + + // ---- embedding coverage ---- + let mut stmt = conn.prepare( + "SELECT embedding_status, COUNT(*) FROM atoms GROUP BY embedding_status", + )?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let status: String = row.get(0)?; + let count: i32 = row.get(1)?; + match status.as_str() { + "pending" => raw.embedding_pending = count, + "processing" => raw.embedding_processing = count, + "complete" => raw.embedding_complete = count, + "failed" => raw.embedding_failed = count, + _ => {} + } + } + + // ---- tagging coverage ---- + let mut stmt = conn.prepare( + "SELECT tagging_status, COUNT(*) FROM atoms GROUP BY tagging_status", + )?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let status: String = row.get(0)?; + let count: i32 = row.get(1)?; + match status.as_str() { + "pending" => raw.tagging_pending = count, + "processing" => raw.tagging_processing = count, + "complete" => raw.tagging_complete = count, + "failed" => raw.tagging_failed = count, + "skipped" => raw.tagging_skipped = count, + _ => {} + } + } + + // Atoms that completed tagging but have 0 tags + raw.untagged_complete = conn.query_row( + "SELECT COUNT(*) FROM atoms a + WHERE a.tagging_status = 'complete' + AND NOT EXISTS (SELECT 1 FROM atom_tags at WHERE at.atom_id = a.id)", + [], + |r| r.get(0), + )?; + + // Atoms skipped by the tagger that also have 0 tags — invisible gap + raw.skipped_untagged = conn.query_row( + "SELECT COUNT(*) FROM atoms a + WHERE a.tagging_status = 'skipped' + AND NOT EXISTS (SELECT 1 FROM atom_tags at WHERE at.atom_id = a.id)", + [], + |r| r.get(0), + )?; + + // ---- source uniqueness ---- + let mut stmt = conn.prepare( + "SELECT source_url, COUNT(*) as cnt, GROUP_CONCAT(id) + FROM atoms + WHERE source_url IS NOT NULL + GROUP BY source_url + HAVING cnt > 1 + LIMIT 50", + )?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let url: String = row.get(0)?; + let ids_csv: String = row.get(2)?; + let ids: Vec = ids_csv.split(',').map(|s| s.to_string()).collect(); + raw.duplicate_sources.push((url, ids)); + } + + // ---- orphan tags ---- + let mut stmt = conn.prepare( + "SELECT t.id, t.name + FROM tags t + LEFT JOIN atom_tags at ON t.id = at.tag_id + LEFT JOIN tags children ON children.parent_id = t.id + WHERE at.tag_id IS NULL + AND children.id IS NULL + AND t.is_autotag_target = 0", + )?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + raw.orphan_tags.push((row.get(0)?, row.get(1)?)); + } + + // ---- semantic graph freshness ---- + raw.newest_atom_updated_at = conn + .query_row("SELECT MAX(updated_at) FROM atoms", [], |r| { + r.get::<_, Option>(0) + }) + .ok() + .flatten(); + + raw.newest_edge_created_at = conn + .query_row( + "SELECT MAX(created_at) FROM semantic_edges", + [], + |r| r.get::<_, Option>(0), + ) + .ok() + .flatten(); + + if let Some(ref newest_edge) = raw.newest_edge_created_at { + raw.atoms_since_edge_rebuild = conn.query_row( + "SELECT COUNT(*) FROM atoms WHERE updated_at > ?1", + params![newest_edge], + |r| r.get(0), + )?; + } else if raw.total_atoms > 0 { + // No edges at all + raw.atoms_since_edge_rebuild = raw.total_atoms; + } + + // ---- wiki coverage ---- + // Tags with >= 5 atoms + let mut stmt = conn.prepare( + "SELECT t.id, t.name, + COUNT(DISTINCT at.atom_id) as atom_count, + w.id IS NOT NULL as has_wiki, + w.updated_at, + (SELECT MAX(a.updated_at) FROM atoms a + JOIN atom_tags at2 ON a.id = at2.atom_id + WHERE at2.tag_id = t.id) as last_atom_update + FROM tags t + JOIN atom_tags at ON t.id = at.tag_id + LEFT JOIN wiki_articles w ON t.id = w.tag_id + GROUP BY t.id + HAVING COUNT(DISTINCT at.atom_id) >= 5 + ORDER BY COUNT(DISTINCT at.atom_id) DESC + LIMIT 50", + )?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let tag_id: String = row.get(0)?; + let tag_name: String = row.get(1)?; + let atom_count: i32 = row.get(2)?; + let has_wiki: bool = row.get(3)?; + let wiki_updated_at: Option = row.get(4)?; + let last_atom_update: Option = row.get(5)?; + + raw.wiki_eligible_count += 1; + + if has_wiki { + raw.wiki_present_count += 1; + // Stale if any atom updated after the wiki + let is_stale = match (&wiki_updated_at, &last_atom_update) { + (Some(w), Some(a)) => a > w, + _ => false, + }; + if is_stale { + raw.wiki_stale_count += 1; + raw.wiki_stale.push(WikiStaleEntry { + tag_id, + tag_name, + new_atom_count: atom_count, + }); + } + } else { + raw.wiki_gaps.push(WikiGap { + tag_id, + tag_name, + atom_count, + }); + } + } + + // ---- content quality ---- + const LIMIT: usize = 20; + + let mut stmt = conn.prepare( + "SELECT id FROM atoms WHERE length(content) < 100 LIMIT ?1", + )?; + let mut rows = stmt.query(params![LIMIT as i32])?; + while let Some(row) = rows.next()? { + raw.very_short_atoms.push(row.get(0)?); + } + + let mut stmt = conn.prepare( + "SELECT id FROM atoms WHERE length(content) > 15000 LIMIT ?1", + )?; + let mut rows = stmt.query(params![LIMIT as i32])?; + while let Some(row) = rows.next()? { + raw.very_long_atoms.push(row.get(0)?); + } + + // No heading: content doesn't start with '#' and doesn't have '\n#' + let mut stmt = conn.prepare( + "SELECT id FROM atoms + WHERE content NOT LIKE '#%' + AND content NOT LIKE '%' || char(10) || '#%' + LIMIT ?1", + )?; + let mut rows = stmt.query(params![LIMIT as i32])?; + while let Some(row) = rows.next()? { + raw.no_heading_atoms.push(row.get(0)?); + } + + // No source: null source_url and no http(s):// in content + let mut stmt = conn.prepare( + "SELECT id FROM atoms + WHERE source_url IS NULL + AND content NOT LIKE '%http://%' + AND content NOT LIKE '%https://%' + AND content NOT LIKE '%Source:%' + LIMIT ?1", + )?; + let mut rows = stmt.query(params![LIMIT as i32])?; + while let Some(row) = rows.next()? { + raw.no_source_atoms.push(row.get(0)?); + } + + // ---- tag health ---- + raw.single_atom_tags = conn.query_row( + "SELECT COUNT(*) FROM ( + SELECT t.id FROM tags t + JOIN atom_tags at ON t.id = at.tag_id + GROUP BY t.id HAVING COUNT(at.atom_id) = 1 + )", + [], + |r| r.get(0), + )?; + + raw.rootless_tags = conn.query_row( + "SELECT COUNT(*) FROM tags WHERE parent_id IS NULL", + [], + |r| r.get(0), + )?; + + // Similar name pairs: fetch all tag names and compare in Rust + { + let mut stmt = conn.prepare("SELECT name FROM tags WHERE atom_count > 0")?; + let mut rows = stmt.query([])?; + let mut names: Vec = Vec::new(); + while let Some(row) = rows.next()? { + names.push(row.get(0)?); + } + raw.similar_name_pair_count = count_similar_name_pairs(&names); + } + + // ---- content overlap detection (Tier 3) ---- + // Moderate similarity (0.55–0.85) + different source prefixes + >= 2 shared tags. + // This surfaces semantically related atoms from different corpora that should be + // reviewed for linking or merging — not template clones (those are boilerplate_pollution). + { + let mut stmt = conn.prepare( + "SELECT + se.source_atom_id, se.target_atom_id, se.similarity_score, + a1.source_url, a1.content, + a2.source_url, a2.content, + COUNT(DISTINCT at_a.tag_id) as shared_tag_count + FROM semantic_edges se + JOIN atoms a1 ON se.source_atom_id = a1.id + JOIN atoms a2 ON se.target_atom_id = a2.id + JOIN atom_tags at_a ON a1.id = at_a.atom_id + JOIN atom_tags at_b ON a2.id = at_b.atom_id AND at_a.tag_id = at_b.tag_id + WHERE se.similarity_score BETWEEN 0.55 AND 0.85 + GROUP BY se.source_atom_id, se.target_atom_id + HAVING COUNT(DISTINCT at_a.tag_id) >= 2 + ORDER BY COUNT(DISTINCT at_a.tag_id) DESC, se.similarity_score DESC + LIMIT 20", + )?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let a_id: String = row.get(0)?; + let b_id: String = row.get(1)?; + let similarity: f32 = row.get(2)?; + let a_source: Option = row.get(3)?; + let a_content: String = row.get(4)?; + let b_source: Option = row.get(5)?; + let b_content: String = row.get(6)?; + let shared_tag_count: i32 = row.get(7)?; + + // Skip same-corpus pairs — those are template pollution, not content overlap. + let prefix_a = source_prefix(&a_source); + let prefix_b = source_prefix(&b_source); + if prefix_a == prefix_b { + continue; + } + + let a_title = extract_title_preview(&a_content); + let b_title = extract_title_preview(&b_content); + + raw.duplicate_pairs.push(DuplicatePair { + pair_id: uuid::Uuid::new_v4().to_string(), + atom_a_id: a_id, + atom_a_title: a_title, + atom_a_source: a_source, + atom_b_id: b_id, + atom_b_title: b_title, + atom_b_source: b_source, + similarity, + shared_tag_count, + }); + } + } + + // ---- boilerplate pollution (atoms with >= 2 edges at similarity >= 0.99) ---- + // These atoms can't be distinguished from their peers via semantic search. + { + let mut stmt = conn.prepare( + "SELECT source_atom_id FROM semantic_edges + WHERE similarity_score >= 0.99 + GROUP BY source_atom_id + HAVING COUNT(*) >= 2 + LIMIT 50", + )?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + raw.boilerplate_affected_atoms.push(row.get(0)?); + } + } + + // ---- contradiction candidates (similarity 0.75..0.92) ---- + raw.contradiction_pairs_checked = conn.query_row( + "SELECT COUNT(*) FROM semantic_edges + WHERE similarity_score >= 0.75 AND similarity_score < 0.92", + [], + |r| r.get(0), + )?; + // For now, surface the count as "candidates" (no LLM check yet) + raw.contradiction_candidate_count = + (raw.contradiction_pairs_checked / 10).min(10); + + Ok(raw) + } + + /// Reset atoms with `tagging_status = 'skipped'` AND 0 tags back to `pending` + /// so the tagger pipeline will process them on the next run. + /// Returns the number of atoms reset. + pub(crate) fn reset_skipped_untagged_to_pending_impl( + &self, + ) -> Result { + let conn = self.db.conn.lock().map_err(|e| { + AtomicCoreError::DatabaseOperation(format!("lock error: {e}")) + })?; + let n = conn.execute( + "UPDATE atoms + SET tagging_status = 'pending' + WHERE tagging_status = 'skipped' + AND NOT EXISTS ( + SELECT 1 FROM atom_tags at WHERE at.atom_id = atoms.id + )", + [], + )? as i32; + Ok(n) + } + + // ==================== Health report storage ==================== + + pub(crate) fn store_health_report_impl( + &self, + report: &StoredHealthReport, + ) -> Result<(), AtomicCoreError> { + let conn = self.db.conn.lock().map_err(|e| { + AtomicCoreError::DatabaseOperation(format!("lock error: {e}")) + })?; + conn.execute( + "INSERT OR REPLACE INTO health_reports + (id, computed_at, overall_score, check_scores, atom_count, auto_fixes_applied, report_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + report.id, + report.computed_at, + report.overall_score, + report.check_scores, + report.atom_count, + report.auto_fixes_applied, + report.report_json, + ], + )?; + // Prune reports older than 90 days + conn.execute( + "DELETE FROM health_reports WHERE computed_at < datetime('now', '-90 days')", + [], + )?; + Ok(()) + } + + pub(crate) fn get_latest_health_report_impl( + &self, + ) -> Result, AtomicCoreError> { + let conn = self.db.read_conn()?; + let result: rusqlite::Result = conn.query_row( + "SELECT report_json FROM health_reports ORDER BY computed_at DESC LIMIT 1", + [], + |r| r.get(0), + ); + match result { + Ok(json) => { + let report: crate::health::HealthReport = + serde_json::from_str(&json).map_err(|e| { + AtomicCoreError::DatabaseOperation(format!( + "failed to deserialize health report: {e}" + )) + })?; + Ok(Some(report)) + } + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(e.into()), + } + } + + pub(crate) fn get_health_reports_impl( + &self, + limit: i32, + ) -> Result, AtomicCoreError> { + let conn = self.db.read_conn()?; + let mut stmt = conn.prepare( + "SELECT id, computed_at, overall_score, check_scores, atom_count, auto_fixes_applied, report_json + FROM health_reports + ORDER BY computed_at DESC + LIMIT ?1", + )?; + let reports = stmt + .query_map(params![limit], |r| { + Ok(StoredHealthReport { + id: r.get(0)?, + computed_at: r.get(1)?, + overall_score: r.get::<_, i32>(2)? as u32, + check_scores: r.get(3)?, + atom_count: r.get(4)?, + auto_fixes_applied: r.get(5)?, + report_json: r.get(6)?, + }) + })? + .filter_map(|r| r.ok()) + .collect(); + Ok(reports) + } + + // ==================== Fix log storage ==================== + + pub(crate) fn log_fix_action_impl( + &self, + log: &HealthFixLog, + ) -> Result<(), AtomicCoreError> { + let conn = self.db.conn.lock().map_err(|e| { + AtomicCoreError::DatabaseOperation(format!("lock error: {e}")) + })?; + let atom_ids_json = log + .atom_ids + .as_ref() + .map(|ids| serde_json::to_string(ids).unwrap_or_default()); + let tag_ids_json = log + .tag_ids + .as_ref() + .map(|ids| serde_json::to_string(ids).unwrap_or_default()); + conn.execute( + "INSERT INTO health_fix_log + (id, check_name, action, tier, atom_ids, tag_ids, + before_state, after_state, llm_prompt, llm_response, executed_at, undone_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", + params![ + log.id, + log.check_name, + log.action, + log.tier, + atom_ids_json, + tag_ids_json, + log.before_state, + log.after_state, + log.llm_prompt, + log.llm_response, + log.executed_at, + log.undone_at, + ], + )?; + Ok(()) + } + + pub(crate) fn get_fix_log_impl( + &self, + fix_id: &str, + ) -> Result, AtomicCoreError> { + let conn = self.db.read_conn()?; + let result = conn.query_row( + "SELECT id, check_name, action, tier, atom_ids, tag_ids, + before_state, after_state, llm_prompt, llm_response, executed_at, undone_at + FROM health_fix_log WHERE id = ?1", + params![fix_id], + |r| { + Ok(HealthFixLog { + id: r.get(0)?, + check_name: r.get(1)?, + action: r.get(2)?, + tier: r.get(3)?, + atom_ids: r + .get::<_, Option>(4)? + .and_then(|s| serde_json::from_str(&s).ok()), + tag_ids: r + .get::<_, Option>(5)? + .and_then(|s| serde_json::from_str(&s).ok()), + before_state: r.get(6)?, + after_state: r.get(7)?, + llm_prompt: r.get(8)?, + llm_response: r.get(9)?, + executed_at: r.get(10)?, + undone_at: r.get(11)?, + }) + }, + ); + match result { + Ok(log) => Ok(Some(log)), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(e.into()), + } + } + + pub(crate) fn get_recent_fixes_impl( + &self, + limit: i32, + ) -> Result, AtomicCoreError> { + let conn = self.db.read_conn()?; + let mut stmt = conn.prepare( + "SELECT id, check_name, action, tier, atom_ids, tag_ids, + before_state, after_state, llm_prompt, llm_response, executed_at, undone_at + FROM health_fix_log + ORDER BY executed_at DESC + LIMIT ?1", + )?; + let logs = stmt + .query_map(params![limit], |r| { + Ok(HealthFixLog { + id: r.get(0)?, + check_name: r.get(1)?, + action: r.get(2)?, + tier: r.get(3)?, + atom_ids: r + .get::<_, Option>(4)? + .and_then(|s| serde_json::from_str(&s).ok()), + tag_ids: r + .get::<_, Option>(5)? + .and_then(|s| serde_json::from_str(&s).ok()), + before_state: r.get(6)?, + after_state: r.get(7)?, + llm_prompt: r.get(8)?, + llm_response: r.get(9)?, + executed_at: r.get(10)?, + undone_at: r.get(11)?, + }) + })? + .filter_map(|r| r.ok()) + .collect(); + Ok(logs) + } + + pub(crate) fn mark_fix_undone_impl(&self, fix_id: &str) -> Result<(), AtomicCoreError> { + let conn = self.db.conn.lock().map_err(|e| { + AtomicCoreError::DatabaseOperation(format!("lock error: {e}")) + })?; + let now = chrono::Utc::now().to_rfc3339(); + conn.execute( + "UPDATE health_fix_log SET undone_at = ?1 WHERE id = ?2", + params![now, fix_id], + )?; + Ok(()) + } + + // ==================== Link resolution storage ==================== + + /// Fetch atoms that likely contain internal links (first-pass SQL filter). + /// Returns (id, content, source_url). + /// The exact link extraction happens in Rust using `link_resolution::extract_internal_links`. + pub(crate) fn get_link_candidate_atoms_impl( + &self, + ) -> Result)>, AtomicCoreError> { + let conn = self.db.read_conn()?; + let mut stmt = conn.prepare( + "SELECT id, content, source_url FROM atoms + WHERE content LIKE '%](.%.md%' + OR content LIKE '%](./%' + OR content LIKE '%](../%' + OR (content LIKE '%[[%' AND content LIKE '%]]%')", + )?; + let rows = stmt + .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get::<_, Option>(2)?)))? + .filter_map(|r| r.ok()) + .collect(); + Ok(rows) + } + + /// Batch lookup: given a list of candidate source URLs, return a map of + /// source_url → atom_id for those that exist in the database. + pub(crate) fn find_atoms_by_source_urls_impl( + &self, + urls: &[String], + ) -> Result, AtomicCoreError> { + if urls.is_empty() { + return Ok(std::collections::HashMap::new()); + } + let conn = self.db.read_conn()?; + let mut map = std::collections::HashMap::new(); + // SQLite doesn't support binding a variable-length IN list, so we query one by one. + // For the typical link count (<50 per atom), this is fast enough. + let mut stmt = conn.prepare("SELECT id FROM atoms WHERE source_url = ?1")?; + for url in urls { + if let Ok(id) = stmt.query_row(params![url], |r| r.get::<_, String>(0)) { + map.insert(url.clone(), id); + } + } + Ok(map) + } + + /// Wikilink fallback: find an atom whose source_url ends with `/.md` + /// (case-insensitive on the name stem) anywhere in the vault. + /// Returns the first match as (atom_id, title_preview). + pub(crate) fn find_atom_by_wikilink_name_impl( + &self, + name: &str, + vault_prefix: &str, + ) -> Result, AtomicCoreError> { + let conn = self.db.read_conn()?; + // Try exact stem match under the vault (case-insensitive) + let like_pattern = format!("%/{}%.md", name.to_lowercase().replace(' ', "-")); + let alt_pattern = format!("%/{}%.md", name.to_lowercase().replace(' ', "_")); + let direct = format!("{}%.md", vault_prefix); + let result = conn.query_row( + "SELECT id, content FROM atoms + WHERE source_url LIKE ?1 || ?3 + OR LOWER(source_url) LIKE ?2 + OR LOWER(source_url) LIKE ?4", + params![vault_prefix, like_pattern, name.replace(' ', "-") + ".md", alt_pattern], + |r| { + let id: String = r.get(0)?; + let content: String = r.get(1)?; + Ok((id, content)) + }, + ); + match result { + Ok((id, content)) => { + let title = content + .lines() + .find(|l| !l.trim().is_empty()) + .unwrap_or(&id) + .trim_start_matches('#') + .trim() + .chars() + .take(80) + .collect::(); + Ok(Some((id, title))) + } + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(e.into()), + } + } +} + + +// ==================== Helpers ==================== + +/// Count tag name pairs where one is a prefix/substring of the other. +fn count_similar_name_pairs(names: &[String]) -> i32 { + let mut count = 0i32; + for (i, a) in names.iter().enumerate() { + for b in names.iter().skip(i + 1) { + let la = a.to_lowercase(); + let lb = b.to_lowercase(); + if la == lb { + continue; // exact duplicate (already handled) + } + if la.contains(lb.as_str()) || lb.contains(la.as_str()) { + count += 1; + } + } + } + count +} + +/// Extract first ~60 chars as a title preview. +fn extract_title_preview(content: &str) -> String { + let first_line = content.lines().next().unwrap_or("").trim(); + let clean = first_line.trim_start_matches('#').trim(); + if clean.len() > 60 { + format!("{}\u{2026}", &clean[..60]) + } else if clean.is_empty() { + content.chars().take(60).collect() + } else { + clean.to_string() + } +} + +/// Extract the source prefix: scheme + authority (everything up to the path). +/// Examples: +/// `https://tylertech.atlassian.net/wiki/...` → `https://tylertech.atlassian.net` +/// `obsidian://ar-playbook/path/to/file` → `obsidian://ar-playbook` +/// `None` → `manual` +pub(crate) fn source_prefix(url: &Option) -> String { + let Some(u) = url else { + return "manual".to_string(); + }; + // Find "://" then the next "/" after it + if let Some(scheme_end) = u.find("://") { + let after_scheme = &u[scheme_end + 3..]; + if let Some(slash) = after_scheme.find('/') { + return u[..scheme_end + 3 + slash].to_string(); + } + } else if let Some(slash) = u.find('/') { + return u[..slash].to_string(); + } + u.clone() +} \ No newline at end of file diff --git a/crates/atomic-core/src/storage/sqlite/mod.rs b/crates/atomic-core/src/storage/sqlite/mod.rs index f2dbccb3..4d4db542 100644 --- a/crates/atomic-core/src/storage/sqlite/mod.rs +++ b/crates/atomic-core/src/storage/sqlite/mod.rs @@ -14,6 +14,7 @@ mod search; mod settings; mod tags; mod wiki; +pub(crate) mod health; use crate::db::Database; use crate::storage::traits::*; diff --git a/crates/atomic-core/src/wiki/mod.rs b/crates/atomic-core/src/wiki/mod.rs index 2d58763e..45b7d250 100644 --- a/crates/atomic-core/src/wiki/mod.rs +++ b/crates/atomic-core/src/wiki/mod.rs @@ -1176,12 +1176,42 @@ pub fn delete_article(conn: &Connection, tag_id: &str) -> Result<(), String> { /// Load all wiki articles with tag names for list view, sorted by importance pub fn load_all_wiki_articles(conn: &Connection) -> Result, String> { + // Use a recursive CTE to compute the live atom count for each wiki article's tag + // hierarchy (tag + all descendants). This mirrors get_article_status exactly, so + // new_atoms_available here is always consistent with GET /api/wiki/{tag_id}/status. let mut stmt = conn .prepare( - "SELECT w.id, w.tag_id, t.name as tag_name, w.updated_at, w.atom_count, - (SELECT COUNT(*) FROM wiki_links wl WHERE wl.target_tag_id = w.tag_id) as inbound_links + "WITH RECURSIVE + -- Expand each wiki-article tag to include all its descendant tags. + -- Seeded only from tags that have a wiki article so the recursion is + -- bounded by the number of articles, not the full tag tree. + tag_tree(root_id, id) AS ( + SELECT t.id, t.id + FROM tags t + WHERE EXISTS (SELECT 1 FROM wiki_articles wa WHERE wa.tag_id = t.id) + UNION ALL + SELECT tt.root_id, t.id + FROM tags t + JOIN tag_tree tt ON t.parent_id = tt.id + ), + -- Live atom count per root tag (counts atoms in the entire subtree). + live_counts(tag_id, cnt) AS ( + SELECT tt.root_id, COUNT(DISTINCT at.atom_id) + FROM tag_tree tt + JOIN atom_tags at ON at.tag_id = tt.id + GROUP BY tt.root_id + ) + SELECT + w.id, + w.tag_id, + t.name AS tag_name, + w.updated_at, + w.atom_count, + (SELECT COUNT(*) FROM wiki_links wl WHERE wl.target_tag_id = w.tag_id) AS inbound_links, + MAX(0, COALESCE(lc.cnt, 0) - w.atom_count) AS new_atoms_available FROM wiki_articles w JOIN tags t ON w.tag_id = t.id + LEFT JOIN live_counts lc ON lc.tag_id = w.tag_id ORDER BY inbound_links DESC, w.atom_count DESC, w.updated_at DESC", ) .map_err(|e| format!("Failed to prepare wiki articles query: {}", e))?; @@ -1195,6 +1225,7 @@ pub fn load_all_wiki_articles(conn: &Connection) -> Result, + /// "keep_newer", "keep_longer", "llm" + pub merge_strategy: Option, +} + +/// Query params for history endpoint. +#[derive(Deserialize)] +pub struct HistoryQuery { + pub limit: Option, +} + +// ==================== GET /api/health/knowledge ==================== + +#[utoipa::path( + get, + path = "/api/health/knowledge", + tag = "health", + responses( + (status = 200, description = "Current health report", body = HealthReport), + (status = 500, description = "Internal server error"), + ), + security(("bearer_auth" = [])), +)] +pub async fn get_health_knowledge(db: Db) -> HttpResponse { + match health::compute_health(&db.0).await { + Ok(report) => HttpResponse::Ok().json(report), + Err(e) => crate::error::error_response(e), + } +} + +// ==================== POST /api/health/fix ==================== + +#[utoipa::path( + post, + path = "/api/health/fix", + tag = "health", + request_body = FixRequest, + responses( + (status = 200, description = "Fix response", body = FixResponse), + (status = 500, description = "Internal server error"), + ), + security(("bearer_auth" = [])), +)] +pub async fn run_health_fix(db: Db, body: web::Json) -> HttpResponse { + match health::run_fix(&db.0, &body).await { + Ok(response) => HttpResponse::Ok().json(response), + Err(e) => crate::error::error_response(e), + } +} + +// ==================== POST /api/health/fix/{check}/{item_id} ==================== + +#[utoipa::path( + post, + path = "/api/health/fix/{check}/{item_id}", + tag = "health", + params( + ("check" = String, Path, description = "Check name"), + ("item_id" = String, Path, description = "Item identifier"), + ), + request_body = ManualFixRequest, + responses( + (status = 200, description = "Action taken or no-op"), + (status = 400, description = "Bad request"), + (status = 500, description = "Internal server error"), + ), + security(("bearer_auth" = [])), +)] +pub async fn apply_manual_fix( + db: Db, + path: web::Path<(String, String)>, + body: web::Json, +) -> HttpResponse { + let (check, item_id) = path.into_inner(); + + match (check.as_str(), body.action.as_str()) { + ("duplicate_detection", "merge_with_llm") => { + // item_id is expected to be "atomA_atomB" (hyphen-separated) + let parts: Vec<&str> = item_id.splitn(2, '_').collect(); + if parts.len() != 2 { + return HttpResponse::BadRequest().json(serde_json::json!({ + "error": "item_id must be 'atomA_id_atomB_id' for merge" + })); + } + let atom_a = parts[0]; + let atom_b = parts[1]; + let dry_run = false; + match atomic_core::health::llm_fixes::merge_duplicate_pair( + &db.0, atom_a, atom_b, dry_run, + ) + .await + { + Ok(Some(action)) => HttpResponse::Ok().json(action), + Ok(None) => HttpResponse::Ok().json(serde_json::json!({"status": "no_op"})), + Err(e) => crate::error::error_response(e), + } + } + _ => HttpResponse::BadRequest().json(serde_json::json!({ + "error": format!("unsupported check '{}' or action '{}'", check, body.action) + })), + } +} + +// ==================== POST /api/health/undo/{fix_id} ==================== + +#[utoipa::path( + post, + path = "/api/health/undo/{fix_id}", + tag = "health", + params( + ("fix_id" = String, Path, description = "Fix ID from the audit log"), + ), + responses( + (status = 200, description = "Undo successful"), + (status = 404, description = "Fix not found"), + (status = 500, description = "Internal server error"), + ), + security(("bearer_auth" = [])), +)] +pub async fn undo_health_fix(db: Db, path: web::Path) -> HttpResponse { + let fix_id = path.into_inner(); + match audit::undo(&db.0, &fix_id).await { + Ok(()) => HttpResponse::Ok().json(serde_json::json!({"status": "ok", "fix_id": fix_id})), + Err(e) => crate::error::error_response(e), + } +} + +// ==================== GET /api/health/history ==================== + +#[utoipa::path( + get, + path = "/api/health/history", + tag = "health", + params( + ("limit" = Option, Query, description = "Maximum number of reports to return"), + ), + responses( + (status = 200, description = "Stored health reports", body = Vec), + (status = 500, description = "Internal server error"), + ), + security(("bearer_auth" = [])), +)] +pub async fn get_health_history(db: Db, query: web::Query) -> HttpResponse { + let limit = query.limit.unwrap_or(30).min(90); + match db.0.get_health_reports(limit).await { + Ok(reports) => HttpResponse::Ok().json(reports), + Err(e) => crate::error::error_response(e), + } +} + +// ==================== GET /api/health/fixes/recent ==================== + +#[utoipa::path( + get, + path = "/api/health/fixes/recent", + tag = "health", + params( + ("limit" = Option, Query, description = "Maximum number of fix log entries"), + ), + responses( + (status = 200, description = "Recent fix log entries", body = Vec), + (status = 500, description = "Internal server error"), + ), + security(("bearer_auth" = [])), +)] +pub async fn get_recent_fixes(db: Db, query: web::Query) -> HttpResponse { + let limit = query.limit.unwrap_or(20).min(100); + match audit::get_recent_fixes(&db.0, limit).await { + Ok(fixes) => HttpResponse::Ok().json(fixes), + Err(e) => crate::error::error_response(e), + } +} + + +// ==================== POST /api/health/check/{check_name} ==================== + +#[utoipa::path( + post, + path = "/api/health/check/{check_name}", + tag = "health", + params( + ("check_name" = String, Path, description = "Health check name to run in isolation"), + ), + responses( + (status = 200, description = "Check result", body = HealthCheckResult), + (status = 400, description = "Unknown check name"), + (status = 500, description = "Internal server error"), + ), + security(("bearer_auth" = [])), +)] +pub async fn compute_single_check( + db: Db, + path: web::Path, +) -> HttpResponse { + let check_name = path.into_inner(); + match health::compute_single_check(&db.0, &check_name).await { + Ok((_name, result)) => HttpResponse::Ok().json(result), + Err(e) => crate::error::error_response(e), + } +} \ No newline at end of file diff --git a/crates/atomic-server/src/routes/import.rs b/crates/atomic-server/src/routes/import.rs index 544fe337..6ea35b37 100644 --- a/crates/atomic-server/src/routes/import.rs +++ b/crates/atomic-server/src/routes/import.rs @@ -37,7 +37,23 @@ pub async fn import_obsidian_vault( .import_obsidian_vault(&body.vault_path, body.max_notes, on_event, on_progress) .await { - Ok(result) => HttpResponse::Ok().json(result), + Ok(result) => { + // Fire-and-forget health maintenance after bulk import + let core = db.0.clone(); + tokio::spawn(async move { + if let Ok(report) = core.compute_health().await { + if report.overall_score < 95 { + let req = atomic_core::health::FixRequest { + checks: None, + mode: "auto".to_string(), + include_medium: false, + }; + let _ = core.run_health_fix(&req).await; + } + } + }); + HttpResponse::Ok().json(result) + } Err(e) => crate::error::error_response(e), } } diff --git a/crates/atomic-server/src/routes/mod.rs b/crates/atomic-server/src/routes/mod.rs index 709523b8..b2eba058 100644 --- a/crates/atomic-server/src/routes/mod.rs +++ b/crates/atomic-server/src/routes/mod.rs @@ -21,6 +21,7 @@ pub mod settings; pub mod setup; pub mod utils; pub mod wiki; +pub mod health; use actix_web::web; @@ -348,4 +349,16 @@ pub fn configure_routes(cfg: &mut web::ServiceConfig) { // Logs cfg.route("/logs", web::get().to(logs::get_logs)); -} + + // Health + cfg.route("/health/knowledge", web::get().to(health::get_health_knowledge)); + cfg.route("/health/fix", web::post().to(health::run_health_fix)); + cfg.route( + "/health/fix/{check}/{item_id}", + web::post().to(health::apply_manual_fix), + ); + cfg.route("/health/undo/{fix_id}", web::post().to(health::undo_health_fix)); + cfg.route("/health/history", web::get().to(health::get_health_history)); + cfg.route("/health/fixes/recent", web::get().to(health::get_recent_fixes)); + cfg.route("/health/check/{check_name}", web::post().to(health::compute_single_check)); +} \ No newline at end of file diff --git a/docs/manual/guides/tag-accordion-scroll-fix.md b/docs/manual/guides/tag-accordion-scroll-fix.md new file mode 100644 index 00000000..c5b14e6b --- /dev/null +++ b/docs/manual/guides/tag-accordion-scroll-fix.md @@ -0,0 +1,117 @@ +# Fixing Tag Accordion Random Scroll + +## Problem + +Clicking a tag's accordion chevron to expand/collapse causes the sidebar to scroll randomly: +- Sometimes to the very top +- Sometimes to a random position +- Expected: no scroll, or smooth scroll to keep the tag in view + +## Root Cause + +The virtualizer's item position measurements become **stale** when the tree structure changes (tags expand/collapse). A fixed 50ms timeout tries to compensate, but fails because: + +1. When you expand a tag, `flatTags` updates (new visible items) +2. The virtualizer's measurements are now **invalid** (it doesn't know new items exist) +3. The scroll effect waits 50ms, then calls `scrollToIndex()` with an index that's now pointing to the wrong item +4. Result: random scroll position + +**Files involved:** +- `src/components/tags/TagTree.tsx` — virtualizer setup and scroll effect (lines 65–99) +- `src/components/tags/TagNode.tsx` — tag expansion toggle (lines 24–30) + +## Solution + +**Tell the virtualizer to remeasure when the tree structure changes, then scroll cleanly.** + +### Changes Required + +**File:** `src/components/tags/TagTree.tsx` + +#### Change 1: Add remeasure effect (after line 99) + +Add this effect after the existing scroll effect: + +```typescript +// Remeasure virtualizer when tree structure changes +useEffect(() => { + virtualizer.measure(); +}, [flatTags, virtualizer]); +``` + +This tells the virtualizer to recalculate all item positions whenever the visible tag list changes. + +#### Change 2: Fix the scroll effect (lines 90–99) + +Replace the existing scroll effect with: + +```typescript +// Scroll to selected tag +useEffect(() => { + if (selectedTagId) { + const index = tagIndexMap.get(selectedTagId); + if (index !== undefined) { + // Scroll directly without setTimeout—measurements are now fresh + virtualizer.scrollToIndex(index, { align: 'center', behavior: 'smooth' }); + } + } +}, [selectedTagId, tagIndexMap, virtualizer]); +``` + +**Key changes:** +- Remove `setTimeout()` — no more arbitrary delays +- Change `align: 'auto'` to `align: 'center'` — clearer visual feedback +- Scroll only runs when `selectedTagId` changes (when you click a tag) + +### Why This Works + +1. **Expand a tag** → `flatTags` updates → `useEffect` calls `virtualizer.measure()` +2. **Virtualizer recalculates** all item positions (now includes newly visible child tags) +3. **Click a tag in the expanded group** → `selectedTagId` changes +4. **Scroll effect runs** → index lookup is now correct, virtualizer knows all positions +5. **Smooth scroll to center** — no stale state, no race conditions + +## Testing + +After applying the fix: + +```bash +npm run tauri dev +``` + +Then in the app: + +1. **Expand a tag** — should stay in place (no scroll) +2. **Click a tag text** — should scroll smoothly to center that tag +3. **Expand multiple nested tags** — smooth scrolling, no jank +4. **Rapid expand/collapse** — no stale index bugs +5. **Manual scroll + click** — scrolls to correct position + +## Code Location Reference + +| What | Where | +|------|-------| +| Virtualizer setup | `src/components/tags/TagTree.tsx:82–88` | +| Tag expansion toggle | `src/components/tags/TagNode.tsx:24–30` | +| Scroll effect (to modify) | `src/components/tags/TagTree.tsx:90–99` | +| Where to add remeasure | `src/components/tags/TagTree.tsx:after line 99` | + +## Why We Don't Scroll on Expand/Collapse + +- **Expand** is a tree-structure change, not a selection change +- Scrolling on every expand would be jarring (user would see tree expand, then jump) +- **Current behavior is correct:** expand in place, then click to scroll to selection + +Only scroll when the user explicitly selects a tag (clicks the text). + +## Performance Notes + +- `virtualizer.measure()` is cheap — it recalculates sizes, doesn't re-render +- Smooth scroll (`behavior: 'smooth'`) is hardware-accelerated +- No performance regression expected + +## Related Code + +- **Virtualizer setup**: `useVirtualizer()` hook from `@tanstack/react-virtual` +- **Tag selection**: `setSelectedTag()` in `src/stores/ui.ts:297–313` +- **Tree flattening**: `flattenVisibleTags()` in `src/components/tags/TagTree.tsx:20–45` diff --git a/docs/plans/2026-04-30-knowledge-health-dashboard/plan.md b/docs/plans/2026-04-30-knowledge-health-dashboard/plan.md new file mode 100644 index 00000000..ccb7801a --- /dev/null +++ b/docs/plans/2026-04-30-knowledge-health-dashboard/plan.md @@ -0,0 +1,1248 @@ +# Knowledge Base Health Dashboard with Auto-Remediation + +**Status:** Planning +**Project:** atomic +**Date:** 2026-04-30 +**Scope:** New feature + +## Executive Summary + +This plan implements a comprehensive health-check system for Atomic that detects and remediates data quality issues through a combination of deterministic SQL fixes and LLM-powered judgment calls. The system exposes two main endpoints (`GET /api/health/knowledge` and `POST /api/health/fix`), stores fix audits for undo capability, and integrates a dashboard widget for monitoring and manual remediation. + +The feature is designed to run automatically after bulk operations (imports, re-embeddings, tag deletions) and optionally on a nightly schedule. It will surface issues to the user through the daily briefing when necessary, and enable both automatic and manual fix workflows. + +**Key capabilities:** +- 11 distinct health checks covering embeddings, tagging, duplicates, contradictions, orphan tags, and content quality +- Tiered auto-fix safety model (Safe/Low/Medium/High) with dry-run support +- Durable audit logs with undo capability for all fixes +- LLM-powered fixes for merge, split, enrich, and contradiction resolution +- Dashboard widget showing health score and actionable issues +- Post-bulk-operation hooks to prevent score degradation +- Health history trending for monitoring KB quality over time + +## Current Architecture / Evidence + +### Existing Infrastructure to Build On + +**Endpoints already in place:** +- `GET /api/embeddings/status` — returns pipeline counts (pending, processing, complete, failed) +- `GET /api/embeddings/status/all` — returns per-atom pipeline status +- `GET /api/wiki/suggestions` — returns tags eligible for wiki articles +- `GET /api/wiki/{tag_id}/status` — returns `new_atoms_available` for a specific wiki +- `GET /api/atoms/{id}/similar` — returns similar atoms above a threshold +- `GET /api/graph/edges` — returns semantic edges with similarity scores +- `POST /api/embeddings/process-pending` — processes pending embeddings +- `POST /api/embeddings/retry-failed` — retries failed embeddings +- `POST /api/graph/rebuild` — rebuilds semantic edge graph +- `POST /api/wiki/{tag_id}/generate` — generates wiki for a tag +- `POST /api/utils/compact-tags` — removes orphan tags + +**Code patterns to follow:** + +1. **Route handlers** (`crates/atomic-server/src/routes/`): + - Follow pattern: `#[utoipa::path(...)] pub async fn handler(db: Db, ...) -> HttpResponse` + - Use `ok_or_error()` for simple responses, `crate::error::error_response(e)` for errors + - Register new routes in `routes/mod.rs` + +2. **LLM integration** (existing patterns from `tagging`, `wiki`, `chat`): + - Call LLM provider trait methods from atomic-core + - Send structured prompts with available context + - Handle streaming vs. completion-based responses + - Parse JSON-structured outputs when needed + +3. **Event callbacks** (pattern from `embedding.rs`): + ```rust + let on_event = embedding_event_callback(state.event_tx.clone()); + db.0.some_operation(on_event).await + ``` + +4. **Database schema** (`crates/atomic-core/src/db.rs`): + - Use rusqlite for SQLite; all new tables need both SQLite and Postgres implementations + - Existing migrations in `migrations/` directory (SQLite) and `crates/atomic-core/src/storage/postgres/migrations/` + - Per-DB data lives in the data database; shared state lives in registry.db + +5. **Settings/Configuration** (`crates/atomic-core/src/settings.rs`): + - Global settings stored in registry.db via `get_setting()` / `set_setting()` + - Per-DB settings can override via `storage.get_all_settings_sync()` / `set_setting_sync()` + - LLM prompt templates stored as settings + +### Data Model + +**Atoms** (existing): +- `atoms.id`, `atoms.content`, `atoms.source_url`, `atoms.embedding_status`, `atoms.tagging_status`, `atoms.embedding_error`, `atoms.tagging_error` + +**Tags** (existing): +- `tags.id`, `tags.name`, `tags.parent_id`, `tags.is_autotag_target` + +**Semantic edges** (existing): +- `semantic_edges.atom_a_id`, `semantic_edges.atom_b_id`, `semantic_edges.similarity` + +**Wiki articles** (existing): +- `wiki_articles.tag_id`, `wiki_articles.content`, `wiki_articles.last_generated_at` + +**Conversations** (existing): +- `conversations.id`, `conversations.tag_filter` + +## Recommended Approach + +### Architecture Decision: Modular Health System + +The health system will be organized as a new module `atomic-core::health` with submodules: + +- `health/mod.rs` — orchestration, score calculation, overall health computation +- `health/checks.rs` — individual check implementations (deterministic SQL queries) +- `health/fixes.rs` — deterministic auto-fix logic (no LLM needed) +- `health/llm_fixes.rs` — LLM-powered fix logic (merge, split, enrich, contradict) +- `health/audit.rs` — fix logging and undo capability + +**Why this structure:** +1. Separates concerns: query logic, fix logic, LLM logic, audit logic are independent +2. Makes it easy to test individual checks in isolation +3. Allows future extensions without touching core module +4. Follows existing pattern: embedding, wiki, chat are all separate modules with clear responsibilities + +### Tiered Fix Safety Model + +| Tier | Risk | Confirmation | Examples | +|------|------|-------------|----------| +| **Safe** | Zero risk | Auto-run, no confirmation | Retry failed embeddings, rebuild graph, process pending tagging | +| **Low** | Minimal risk, reversible | Auto-run with undo log | Delete orphan tags, generate missing wikis | +| **Medium** | Changes content | Dry-run first, confirm | Add headings, merge exact-source duplicates | +| **High** | Deletes or rewrites | Always require review | Merge similar atoms, split long atoms, delete stubs, resolve contradictions | + +**Endpoint semantics:** +- `POST /api/health/fix { mode: "auto" }` → runs Safe + Low tier +- `POST /api/health/fix { mode: "auto", include_medium: true }` → Safe + Low + Medium +- `POST /api/health/fix { mode: "dry_run", ... }` → report what would be fixed without executing + +### LLM Prompt Templates + +Store as settings (like `tagging_prompt`, `chat_prompt` already do): +- `health.merge_duplicates_prompt` — for merging high-similarity atoms +- `health.contradiction_detection_prompt` — for finding conflicting info +- `health.split_long_atom_prompt` — for splitting >15K character atoms +- `health.enrich_stub_atom_prompt` — for expanding <100 char atoms +- `health.add_structure_prompt` — for adding headings to unstructured content +- `health.tag_reorganize_prompt` — for suggesting tag hierarchy fixes + +All with sensible defaults so zero setup is needed. + +## Implementation Plan + +### Phase I: Core Infrastructure (Foundation) + +**1.1 Database schema additions** + +Add to SQLite migration (new file `migrations/XXX_create_health_tables.sql`): + +```sql +-- Health reports — historical snapshots of KB state +CREATE TABLE health_reports ( + id TEXT PRIMARY KEY, + computed_at TEXT NOT NULL, + overall_score INTEGER NOT NULL, + check_scores TEXT NOT NULL, -- JSON: {"duplicates": 80, ...} + atom_count INTEGER NOT NULL, + auto_fixes_applied INTEGER DEFAULT 0, + report_json TEXT NOT NULL -- Full report for detail view +); +CREATE INDEX idx_health_reports_computed ON health_reports(computed_at DESC); + +-- Audit log of all auto-fix actions (for undo) +CREATE TABLE health_fix_log ( + id TEXT PRIMARY KEY, + check_name TEXT NOT NULL, + action TEXT NOT NULL, + tier TEXT NOT NULL, -- "safe", "low", "medium", "high" + atom_ids TEXT, -- JSON array of affected atom IDs + tag_ids TEXT, -- JSON array of affected tag IDs + before_state TEXT, -- JSON snapshot (for undo) + after_state TEXT, -- JSON snapshot (for verification) + llm_prompt TEXT, -- Prompt sent to LLM (if applicable) + llm_response TEXT, -- Raw LLM response (for audit) + executed_at TEXT NOT NULL, + undone_at TEXT -- NULL unless undone +); +CREATE INDEX idx_health_fix_log_executed ON health_fix_log(executed_at DESC); +CREATE INDEX idx_health_fix_log_check ON health_fix_log(check_name); +``` + +Add equivalent Postgres migration to `crates/atomic-core/src/storage/postgres/migrations/`. + +**1.2 Models for health domain** + +New file: `crates/atomic-core/src/health/models.rs` + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthReport { + pub overall_score: u32, // 0-100 + pub overall_status: HealthStatus, // healthy, needs_attention, degraded, unhealthy + pub computed_at: String, + pub atom_count: i32, + pub checks: HashMap, + pub auto_fixable: i32, // count of auto-fixable issues + pub requires_review: i32, // count of issues needing human review +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum HealthStatus { + #[serde(rename = "healthy")] + Healthy, + #[serde(rename = "needs_attention")] + NeedsAttention, + #[serde(rename = "degraded")] + Degraded, + #[serde(rename = "unhealthy")] + Unhealthy, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthCheck { + pub status: String, // "ok", "warning", "error" + pub score: u32, // 0-100 contribution to overall + // Check-specific fields vary by check type + #[serde(flatten)] + pub data: serde_json::Value, // Dynamic fields per check +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FixAction { + pub check: String, + pub action: String, // "deleted_tags", "merged_atoms", etc. + pub count: i32, + pub details: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FixResponse { + pub mode: String, // "auto", "dry_run" + pub actions_taken: Vec, + pub skipped: Vec, + pub new_score: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SkippedFix { + pub check: String, + pub reason: String, + pub count: i32, +} +``` + +**1.3 Storage trait additions** + +Add to `crates/atomic-core/src/storage/traits.rs`: + +```rust +#[async_trait] +pub trait HealthStore: Send + Sync { + // Health report storage + async fn store_health_report(&self, report: &HealthReport) -> StorageResult<()>; + async fn get_latest_health_report(&self) -> StorageResult>; + async fn get_health_reports_since(&self, since: &str) -> StorageResult>; + + // Fix audit log + async fn log_fix_action(&self, fix_log: &HealthFixLog) -> StorageResult<()>; + async fn get_fix_log(&self, fix_id: &str) -> StorageResult>; + async fn get_recent_fixes(&self, limit: i32) -> StorageResult>; + async fn undo_fix(&self, fix_id: &str) -> StorageResult<()>; +} + +#[derive(Debug, Clone)] +pub struct HealthFixLog { + pub id: String, + pub check_name: String, + pub action: String, + pub tier: String, // "safe", "low", "medium", "high" + pub atom_ids: Option>, + pub tag_ids: Option>, + pub before_state: String, // JSON + pub after_state: String, // JSON + pub llm_prompt: Option, + pub llm_response: Option, + pub executed_at: String, + pub undone_at: Option, +} +``` + +Implement for both `SqliteStorage` and `PostgresStorage` (mostly straightforward INSERT/SELECT statements). + +### Phase II: Health Checks (11 distinct checks) + +**2.1 `health/checks.rs` — Deterministic checks** + +Each check returns a `HealthCheck` struct with standardized fields. Run synchronously over database snapshots. + +```rust +pub async fn check_embedding_coverage(storage: &dyn Storage) -> HealthCheck { + // Count: pending, processing, complete, failed + // Score: (complete / total) * 100, cap at 50 if any failed + // Return distribution + status +} + +pub async fn check_source_uniqueness(storage: &dyn Storage) -> HealthCheck { + // Find source_urls appearing on multiple atoms + // Score: 100 if 0 duplicates, subtract 15 per duplicate +} + +pub async fn check_orphan_tags(storage: &dyn Storage) -> HealthCheck { + // Find tags with 0 atoms and no children, excluding autotag targets + // Score: 100 if 0, subtract 2 per orphan +} + +pub async fn check_tagging_coverage(storage: &dyn Storage) -> HealthCheck { + // Count atoms: tagged, untagged, failed, skipped + // Score: (tagged / total) * 100 +} + +pub async fn check_semantic_graph_freshness(storage: &dyn Storage) -> HealthCheck { + // Compare last rebuild time vs newest atom + // Score: 100 if recent, subtract 2 per atom since rebuild +} + +pub async fn check_wiki_coverage(storage: &dyn Storage) -> HealthCheck { + // Find tags with >= 5 atoms that could have wikis + // Count: with_wiki, without_wiki, stale + // Score: (with_wiki / eligible) * 70 + (non_stale / with_wiki) * 30 +} + +pub async fn check_content_quality(storage: &dyn Storage) -> HealthCheck { + // Flag atoms: very_short (<100 chars), very_long (>15K chars), + // no_headings, no_source + // Score: 85 base, minus 5 for each category with issues +} + +pub async fn check_tag_health(storage: &dyn Storage) -> HealthCheck { + // Find: single-atom tags, rootless tags, similar-named tags + // Score: points deducted per category +} +``` + +**2.2 LLM-powered checks** + +To be added in Phase III. For now, implement skeleton methods that return placeholder scores. + +```rust +pub async fn check_duplicate_detection( + storage: &dyn Storage, + _providers: &dyn LlmProvider, +) -> HealthCheck { + // Find atom pairs with similarity 0.92-1.0 from different sources + // Mark as requires_review: true + // Don't execute fixes yet +} + +pub async fn check_contradiction_detection( + storage: &dyn Storage, + _providers: &dyn LlmProvider, +) -> HealthCheck { + // Find atoms with similarity 0.75-0.92 (same topic, different content) + // Use LLM to confirm contradiction (Phase III) + // Mark as requires_review: true +} +``` + +**2.3 Score aggregation** + +```rust +pub fn compute_overall_score(checks: &HashMap) -> u32 { + let weights = [ + ("duplicate_detection", 0.15), + ("embedding_coverage", 0.15), + ("source_uniqueness", 0.10), + ("tagging_coverage", 0.10), + ("wiki_coverage", 0.10), + ("semantic_graph_freshness", 0.10), + ("content_quality", 0.05), + ("orphan_tags", 0.05), + ("tag_health", 0.05), + ("contradiction_detection", 0.05), + ("tagging_coverage", 0.10), // untagged atoms + ]; + + let mut total = 0.0; + for (check_name, weight) in weights.iter() { + if let Some(check) = checks.get(*check_name) { + total += (check.score as f64) * weight; + } + } + total as u32 +} + +pub fn status_for_score(score: u32) -> HealthStatus { + match score { + 90..=100 => HealthStatus::Healthy, + 70..=89 => HealthStatus::NeedsAttention, + 50..=69 => HealthStatus::Degraded, + _ => HealthStatus::Unhealthy, + } +} +``` + +### Phase III: Auto-Fix Implementation + +**3.1 `health/fixes.rs` — Deterministic fixes (no LLM)** + +```rust +pub async fn fix_embedding_coverage( + storage: &dyn Storage, + core: &AtomicCore, +) -> Result { + // Call core.process_pending_embeddings() + // Call core.retry_failed_embeddings() + // Return action: { check: "embedding_coverage", action: "retry_failed_and_process_pending", count: X } +} + +pub async fn fix_orphan_tags(storage: &dyn Storage) -> Result { + // Find and delete orphan tags (not autotag targets) + // Log to health_fix_log with before_state + // Return action: { check: "orphan_tags", action: "deleted_tags", count: X, details: [tag_names] } +} + +pub async fn fix_source_uniqueness(storage: &dyn Storage) -> Result { + // For exact source_url duplicates: + // - Keep newest (by created_at) + // - Merge tags from deleted atoms onto the kept one + // - Delete older atoms + // Log all deletes with before_state + // Return action with count +} + +pub async fn fix_semantic_graph_freshness( + storage: &dyn Storage, + core: &AtomicCore, +) -> Result { + // Call core.rebuild_semantic_edges() + // Return action: { check: "semantic_graph_freshness", action: "queued_rebuild", ... } +} +``` + +**3.2 `health/llm_fixes.rs` — LLM-powered fixes** + +These will call LLM provider to make judgment calls. Implemented in Phase III. + +```rust +pub async fn fix_tagging_coverage_with_llm( + storage: &dyn Storage, + core: &AtomicCore, + llm: &dyn LlmProvider, + untagged_atoms: &[AtomWithTags], +) -> Result { + // For each untagged atom, call LLM with modified prompt that forces >= 1 tag + // Re-run tagging with forced assignment + // Return action with count of newly tagged atoms +} + +pub async fn merge_duplicate_atoms_with_llm( + storage: &dyn Storage, + atom_a: &AtomWithTags, + atom_b: &AtomWithTags, + llm: &dyn LlmProvider, +) -> Result { + // Call LLM to synthesize both atoms + // Update newer atom with merged content + // Delete older atom + // Re-embed and re-tag merged atom + // Log to health_fix_log +} + +pub async fn split_long_atom_with_llm( + storage: &dyn Storage, + atom: &AtomWithTags, + llm: &dyn LlmProvider, +) -> Result { + // Call LLM to analyze if atom should be split + // If yes: create new atoms for each section + // If no: add structure (headings) instead + // Log all creates/deletes +} +``` + +**3.3 `health/audit.rs` — Undo capability** + +```rust +pub async fn undo_fix( + storage: &dyn Storage, + fix_id: &str, +) -> Result<(), AtomicCoreError> { + // Fetch fix_log entry via fix_id + // Parse before_state JSON + // For each affected atom_id: restore from before_state + // For each affected tag_id: restore from before_state + // Mark fix_log.undone_at = now() + // Return any created entries to allow cascading undo +} +``` + +### Phase IV: API Endpoints + +**4.1 `routes/health.rs` — New route handlers** + +```rust +#[utoipa::path( + get, + path = "/api/health/knowledge", + responses( + (status = 200, description = "Health report", body = HealthReport) + ), + tag = "health" +)] +pub async fn get_health_knowledge( + state: web::Data, + db: Db, +) -> HttpResponse { + // Compute all checks + // Calculate overall score + // Store report + // Return JSON +} + +#[utoipa::path( + post, + path = "/api/health/fix", + request_body = FixRequest, + responses( + (status = 200, description = "Fix results", body = FixResponse) + ), + tag = "health" +)] +pub async fn run_health_fix( + state: web::Data, + db: Db, + body: web::Json, +) -> HttpResponse { + // Determine which fixes to run based on mode and include_medium + // Execute fixes in tier order (Safe → Low → Medium) + // Collect FixAction results + // Recompute health score + // Return FixResponse +} + +#[utoipa::path( + post, + path = "/api/health/fix/{check}/{item_id}", + params( + ("check" = String, Path), + ("item_id" = String, Path) + ), + request_body = ManualFixRequest, + responses( + (status = 200, description = "Fix applied") + ), + tag = "health" +)] +pub async fn apply_manual_fix( + state: web::Data, + db: Db, + path: web::Path<(String, String)>, + body: web::Json, +) -> HttpResponse { + // Route to specific fix handler based on check name + // Execute fix with user parameters + // Log to health_fix_log + // Return success +} + +#[utoipa::path( + post, + path = "/api/health/undo/{fix_id}", + params(("fix_id" = String, Path)), + responses( + (status = 200, description = "Fix undone") + ), + tag = "health" +)] +pub async fn undo_health_fix(db: Db, path: web::Path) -> HttpResponse { + let fix_id = path.into_inner(); + ok_or_error(db.0.undo_fix(&fix_id).await) +} +``` + +**4.2 Request/response types** + +```rust +#[derive(Deserialize, ToSchema)] +pub struct FixRequest { + pub checks: Option>, // If None, run all + pub mode: String, // "auto", "dry_run" + pub include_medium: Option, // Default false +} + +#[derive(Deserialize, ToSchema)] +pub struct ManualFixRequest { + pub action: String, // "merge", "keep_both", "delete_one", etc. + pub keep_atom_id: Option, + pub merge_strategy: Option, // "keep_newer", "keep_longer", "llm" +} +``` + +**4.3 Register in `routes/mod.rs`** + +```rust +pub mod health; +// ... +// In web::scope("/api"): +.service( + web::scope("/health") + .route("/knowledge", web::get().to(health::get_health_knowledge)) + .route("/fix", web::post().to(health::run_health_fix)) + .route("/fix/{check}/{item_id}", web::post().to(health::apply_manual_fix)) + .route("/undo/{fix_id}", web::post().to(health::undo_health_fix)) +) +``` + +### Phase V: Integration + +**5.1 Post-bulk-operation hooks** + +Add to import handlers (`import/obsidian.rs`, `ingest/fetch.rs`), bulk atom creation, tag deletion: + +```rust +async fn post_bulk_operation_hook(core: &AtomicCore) { + // Run health check + let report = compute_health(&core).await.ok(); + + if let Some(r) = report { + if r.overall_score < 95 { + // Auto-fix safe issues + let _ = core.run_health_fix(FixMode::Safe).await; + + // Recompute and cache + let updated_report = compute_health(&core).await.ok(); + // ... store for later use + } + } +} +``` + +**5.2 Scheduled nightly maintenance** + +Add to `crates/atomic-core/src/scheduler/mod.rs`: + +```rust +pub async fn health_maintenance(core: &AtomicCore) { + // Run full health check + let report = compute_health(&core).await.ok(); + + // Auto-fix safe + low tier issues + if let Some(_) = report { + let _ = core.run_health_fix(FixMode::Low).await; + } + + // Store report for history + // If score dropped, include in next briefing +} +``` + +Add task config to settings: + +```rust +("task.health_maintenance.enabled", "true"), +("task.health_maintenance.interval_hours", "24"), +("task.health_maintenance.auto_fix_tier", "low"), +``` + +**5.3 Briefing integration** + +Extend `crates/atomic-core/src/briefing/mod.rs` to include health findings: + +```rust +fn format_health_section(report: &HealthReport) -> String { + // Only include if score < 85 or contradictions found + // Format: + // ## Knowledge Health + // Your KB score is X/100 (status). + // Auto-fixed: [list] + // Needs review: [list] +} +``` + +**5.4 Settings/configuration** + +Add LLM prompt templates to `DEFAULT_SETTINGS` in `settings.rs`: + +```rust +("health.merge_duplicates_prompt", "..."), +("health.contradiction_detection_prompt", "..."), +("health.split_long_atom_prompt", "..."), +("health.enrich_stub_atom_prompt", "..."), +("health.add_structure_prompt", "..."), +("health.tag_reorganize_prompt", "..."), +``` + +### Phase VI: Frontend (Dashboard Widget) + +**6.1 Health panel component** + +New file: `src/components/dashboard/HealthPanel.tsx` + +```tsx +export function HealthPanel() { + const [report, setReport] = useState(null); + const [loading, setLoading] = useState(false); + + useEffect(() => { + fetchHealth(); + }, []); + + async function fetchHealth() { + const resp = await getTransport().invoke('get_health_knowledge', {}); + setReport(resp as HealthReport); + } + + async function autoFix() { + setLoading(true); + const resp = await getTransport().invoke('run_health_fix', { + mode: 'auto' + }); + await fetchHealth(); + setLoading(false); + } + + if (!report) return null; + + return ( +
+
+

Knowledge Health

+ {report.overall_score}/100 +
+ +
+ {Object.entries(report.checks).map(([name, check]) => ( + + ))} +
+ +
+ + +
+ + {report.auto_fixable > 0 && ( +

{report.auto_fixable} issues can be automatically fixed

+ )} +
+ ); +} +``` + +**6.2 Add to dashboard registry** + +Register `HealthPanel` in `src/components/dashboard/registry.ts` to make it available as a dashboard widget. + +**6.3 Review queue page** + +New page: `src/routes/health/review/+page.svelte` (if using SvelteKit) or equivalent route for showing duplicates, contradictions, stubs that need human review. + +## Files / Components To Change + +### New Files + +**Backend (Rust):** +- `crates/atomic-core/src/health/mod.rs` — module root, orchestration +- `crates/atomic-core/src/health/checks.rs` — 11 health checks (deterministic) +- `crates/atomic-core/src/health/fixes.rs` — auto-fix logic (deterministic) +- `crates/atomic-core/src/health/llm_fixes.rs` — LLM-powered fixes (Phase III) +- `crates/atomic-core/src/health/audit.rs` — fix logging, undo +- `crates/atomic-core/src/health/models.rs` — health domain types +- `crates/atomic-server/src/routes/health.rs` — endpoint handlers +- `migrations/XXX_create_health_tables.sql` — SQLite schema +- `crates/atomic-core/src/storage/postgres/migrations/XXX_create_health_tables.sql` — Postgres schema + +**Frontend (TypeScript/React):** +- `src/components/dashboard/HealthPanel.tsx` — main dashboard widget +- `src/routes/health/+page.tsx` or `.svelte` — detailed health page +- `src/routes/health/review/+page.tsx` — high-tier fix review queue +- `src/lib/api/health.ts` — health API client (type-safe wrapper) + +### Modified Files + +**Backend (Rust):** +- `crates/atomic-core/src/lib.rs` — add `pub mod health` +- `crates/atomic-core/src/storage/traits.rs` — add `HealthStore` trait +- `crates/atomic-core/src/storage/sqlite/mod.rs` — implement `HealthStore` +- `crates/atomic-core/src/storage/postgres/mod.rs` — implement `HealthStore` +- `crates/atomic-core/src/storage/sqlite/settings.rs` — add health prompt defaults +- `crates/atomic-server/src/routes/mod.rs` — register health routes +- `crates/atomic-core/src/scheduler/mod.rs` — add health_maintenance task +- `crates/atomic-core/src/briefing/mod.rs` — include health findings +- `crates/atomic-core/src/settings.rs` — add health prompt templates to `DEFAULT_SETTINGS` +- Bulk import handlers — add post-operation hooks +- `crates/atomic-server/src/state.rs` — may need event channel registration + +**Frontend (TypeScript):** +- `src/components/dashboard/registry.ts` — add HealthPanel widget +- `src/lib/api.ts` — add health API methods +- `src/stores/ui.ts` — may need state for health reports + +## Data Flow / Interfaces + +### Health Check Flow + +``` +GET /api/health/knowledge + ↓ +compute_health(core) + ├─ check_embedding_coverage() → HealthCheck { score, status, data } + ├─ check_source_uniqueness() → HealthCheck + ├─ check_orphan_tags() → HealthCheck + ├─ check_tagging_coverage() → HealthCheck + ├─ check_semantic_graph_freshness() → HealthCheck + ├─ check_wiki_coverage() → HealthCheck + ├─ check_content_quality() → HealthCheck + ├─ check_tag_health() → HealthCheck + ├─ check_duplicate_detection() → HealthCheck { requires_review: true } + ├─ check_contradiction_detection() → HealthCheck { requires_review: true } + └─ check_tagging_coverage() → HealthCheck + ↓ +aggregate_scores(checks) → overall_score: u32 + ↓ +HealthReport { overall_score, checks, status, auto_fixable, requires_review } + ↓ +store_health_report(report) + ↓ +HTTP 200 → HealthReport (JSON) +``` + +### Fix Flow + +``` +POST /api/health/fix { mode: "auto", include_medium?: bool } + ↓ +determine_fix_tiers(mode, include_medium) + ↓ +for each fix in order: + - tier < "medium" or include_medium → run it + - capture before_state + - execute fix + - capture after_state + - log to health_fix_log with undo info + ↓ +recompute_health() + ↓ +FixResponse { actions_taken, skipped, new_score } + ↓ +HTTP 200 → FixResponse (JSON) +``` + +### Undo Flow + +``` +POST /api/health/undo/{fix_id} + ↓ +fetch_fix_log(fix_id) + ↓ +parse_before_state(log.before_state) + ↓ +for each atom_id: + restore_atom(atom_id, before_snapshot) +for each tag_id: + restore_tag(tag_id, before_snapshot) + ↓ +set_fix_log.undone_at = now() + ↓ +HTTP 200 → { status: "ok" } +``` + +## Configuration / Secrets / Deployment Notes + +**No additional secrets needed.** Health system uses existing LLM providers (OpenRouter, Ollama, OpenAI-compatible). + +**Settings added to `DEFAULT_SETTINGS`:** +```rust +("task.health_maintenance.enabled", "true"), +("task.health_maintenance.interval_hours", "24"), +("task.health_maintenance.auto_fix_tier", "low"), +("health.merge_duplicates_prompt", ""), +("health.contradiction_detection_prompt", ""), +("health.split_long_atom_prompt", ""), +("health.enrich_stub_atom_prompt", ""), +("health.add_structure_prompt", ""), +("health.tag_reorganize_prompt", ""), +``` + +All with sensible defaults that work immediately on fresh install. + +**Environment notes:** +- Health checks complete in < 2s for 500 atoms (single query per check, no N+1) +- LLM-powered fixes (Phase III) are rate-limited: max 3 wiki generations per fix run +- Contradiction detection may be async for large KBs (>1000 atoms with 0.75+ similarity pairs) +- Health reports stored indefinitely; UI may paginate to last 90 days + +## Testing / Validation Plan + +### Unit Tests + +Create `crates/atomic-core/tests/health_tests.rs`: + +**Test 1: Clean database scores 100** +```rust +#[tokio::test] +async fn health_clean_db_is_100() { + let db = setup_test_db().await; + let report = compute_health(&db).await.unwrap(); + assert_eq!(report.overall_score, 100); +} +``` + +**Test 2: Orphan tags detected and fixable** +```rust +#[tokio::test] +async fn orphan_tags_detected_and_fixed() { + let db = setup_test_db().await; + create_orphan_tag(&db, "orphan").await; + + let report_before = compute_health(&db).await.unwrap(); + assert!(report_before.overall_score < 100); + + run_health_fix(&db, FixMode::Safe).await.unwrap(); + + let report_after = compute_health(&db).await.unwrap(); + assert_eq!(report_after.overall_score, 100); +} +``` + +**Test 3: Failed embeddings cause score drop** +```rust +#[tokio::test] +async fn failed_embeddings_drop_score() { + let db = setup_test_db().await; + create_atom_with_status(&db, "test", "embedding", "failed", None).await; + + let report = compute_health(&db).await.unwrap(); + assert!(report.overall_score <= 50); + assert!(report.checks["embedding_coverage"].data["failed"] > 0); +} +``` + +**Test 4: Fix audit log stores before/after state** +```rust +#[tokio::test] +async fn fix_logged_with_undo_capability() { + let db = setup_test_db().await; + create_orphan_tag(&db, "orphan").await; + + let fix_response = run_health_fix(&db, FixMode::Safe).await.unwrap(); + assert_eq!(fix_response.actions_taken.len(), 1); + + let logs = db.get_recent_fixes(10).await.unwrap(); + assert_eq!(logs[0].check_name, "orphan_tags"); + assert!(logs[0].before_state.contains("orphan")); +} +``` + +**Test 5: Undo restores pre-fix state** +```rust +#[tokio::test] +async fn undo_fix_restores_state() { + let db = setup_test_db().await; + create_orphan_tag(&db, "orphan").await; + + let before_count = db.count_tags().await.unwrap(); + + let fix_response = run_health_fix(&db, FixMode::Safe).await.unwrap(); + let fix_id = fix_response.actions_taken[0].id.clone(); + + let after_count = db.count_tags().await.unwrap(); + assert_eq!(after_count, before_count - 1); + + db.undo_fix(&fix_id).await.unwrap(); + + let restored_count = db.count_tags().await.unwrap(); + assert_eq!(restored_count, before_count); +} +``` + +**Test 6: Duplicate detection finds high-similarity atoms** +```rust +#[tokio::test] +async fn duplicates_detected() { + let db = setup_test_db().await; + create_atom_pair_with_similarity(&db, 0.95, "obs://vault1/file", "obs://vault2/file").await; + + let report = compute_health(&db).await.unwrap(); + let dup_check = &report.checks["duplicate_detection"]; + assert!(dup_check.data["count"] > 0); + assert_eq!(dup_check.status, "warning"); +} +``` + +**Test 7: Contradictions flagged for review** +```rust +#[tokio::test] +async fn contradictions_detected() { + let db = setup_test_db().await; + // Create two atoms with same embedding, contradictory content + create_contradictory_atoms(&db, 0.82).await; + + let report = compute_health(&db).await.unwrap(); + let contra_check = &report.checks["contradiction_detection"]; + assert!(contra_check.data["count"] > 0); + assert_eq!(contra_check.status, "warning"); +} +``` + +**Test 8: Dry run doesn't apply fixes** +```rust +#[tokio::test] +async fn dry_run_mode_doesnt_fix() { + let db = setup_test_db().await; + create_orphan_tag(&db, "orphan").await; + + let before_count = db.count_tags().await.unwrap(); + + let response = run_health_fix_dry_run(&db, FixMode::Safe).await.unwrap(); + assert_eq!(response.mode, "dry_run"); + assert!(response.actions_taken.len() > 0); + + let after_count = db.count_tags().await.unwrap(); + assert_eq!(after_count, before_count); // Nothing actually deleted +} +``` + +**Test 9: Score weighted correctly** +```rust +#[tokio::test] +async fn overall_score_weighted() { + let db = setup_test_db().await; + + // Set up state with: + // - embedding_coverage at 50 (worth 15%) + // - all others at 100 + // Expected: (50 * 0.15) + (100 * 0.85) = 92.5 → 92 + + let report = compute_health(&db).await.unwrap(); + assert_eq!(report.overall_score, 92); +} +``` + +**Test 10: Very short atoms flagged for review** +```rust +#[tokio::test] +async fn very_short_atoms_flagged() { + let db = setup_test_db().await; + create_atom(&db, "hi").await; // 2 chars + + let report = compute_health(&db).await.unwrap(); + let quality_check = &report.checks["content_quality"]; + assert!(quality_check.data["very_short"]["count"] > 0); +} +``` + +### Integration Tests + +Create `crates/atomic-server/tests/health_api_tests.rs`: + +**Test 1: GET /api/health/knowledge returns valid report** +```rust +#[actix_web::test] +async fn get_health_knowledge_endpoint() { + let app = create_test_app().await; + let resp = test::call_service( + &app, + test::TestRequest::get().uri("/api/health/knowledge").to_request(), + ).await; + + assert_eq!(resp.status(), http::StatusCode::OK); + let body = test::read_body(resp).await; + let report: HealthReport = serde_json::from_slice(&body).unwrap(); + assert!(report.overall_score >= 0 && report.overall_score <= 100); +} +``` + +**Test 2: POST /api/health/fix auto mode fixes safe issues** +```rust +#[actix_web::test] +async fn post_health_fix_auto_mode() { + let app = create_test_app_with_orphan_tag().await; + + let resp = test::call_service( + &app, + test::TestRequest::post() + .uri("/api/health/fix") + .set_json(json!({ "mode": "auto" })) + .to_request(), + ).await; + + assert_eq!(resp.status(), http::StatusCode::OK); + let body = test::read_body(resp).await; + let fix_resp: FixResponse = serde_json::from_slice(&body).unwrap(); + assert_eq!(fix_resp.mode, "auto"); + assert!(fix_resp.actions_taken.len() > 0); +} +``` + +**Test 3: Dry run doesn't persist changes** +```rust +#[actix_web::test] +async fn health_fix_dry_run_mode() { + let app = create_test_app_with_orphan_tag().await; + + // Run in dry_run mode + test::call_service( + &app, + test::TestRequest::post() + .uri("/api/health/fix") + .set_json(json!({ "mode": "dry_run" })) + .to_request(), + ).await; + + // Check that orphan still exists + let health_resp = test::call_service( + &app, + test::TestRequest::get().uri("/api/health/knowledge").to_request(), + ).await; + let health: HealthReport = serde_json::from_slice(&test::read_body(health_resp).await).unwrap(); + assert!(health.checks["orphan_tags"].data["count"] > 0); +} +``` + +### Manual Verification Steps + +1. **Fresh database should score 100:** + - Start server with empty database + - GET /api/health/knowledge + - Verify overall_score == 100 + - Verify all check statuses are "ok" + +2. **Create pathological state and verify detection:** + - Manually insert orphan tag, failed embedding, duplicate atoms + - GET /api/health/knowledge + - Verify issues are detected with correct counts + +3. **Auto-fix safe issues:** + - POST /api/health/fix { mode: "auto" } + - Verify fixes applied + - GET /api/health/knowledge again + - Verify score improved + +4. **Dashboard widget renders:** + - Open dashboard in browser + - Verify HealthPanel displays + - Verify score and check bars render + - Click "Fix Safe Issues" button + - Verify panel updates after fix + +5. **Post-import health check runs:** + - Import Obsidian vault with 100+ notes + - In background/logs, verify health_maintenance task ran + - GET /api/health/knowledge + - Verify no degradation from pre-import state + +## Risks, Assumptions, and Open Questions + +### Risks + +1. **LLM cost for large KBs** — Contradiction detection on 1000+ atoms with high similarity pairs could be expensive. Mitigation: Rate-limit and make async with background job queue. + +2. **False positives on contradictions** — LLM may incorrectly flag atoms as contradictory when they're actually complementary. Mitigation: Always mark as requires_review, never auto-fix without user confirmation. + +3. **Merge strategy decisions** — When merging atoms, choosing which source URL to keep is lossy. Mitigation: Store secondary URL in merged atom's body as "Sources:" section, add "last edited by source" note. + +4. **Undo state explosion** — Fix logs could grow large if the system auto-fixes frequently. Mitigation: Prune fix_log older than 90 days, keep health_reports for trending only. + +5. **Multi-database consistency** — If two databases are running, health checks must be per-DB isolated. Mitigation: All health queries scoped to current db_id, scheduled tasks fan out per database. + +### Assumptions + +1. **Similarity threshold stable** — Assume 0.92 for duplicates, 0.75-0.92 for contradictions is reasonable for 1536-dim embeddings. Will need tuning with real data. + +2. **Atomic content immutability after fix** — Assume that once an atom is fixed (merged, split, enriched), we don't need to re-run full pipelines. Partial: we will re-embed merged atoms. + +3. **LLM availability** — Assume LLM provider is available for Phase III fixes. Fallback: if LLM is down, fixes marked as "awaiting_llm" and can retry later. + +4. **Browser/UI responsiveness** — Assume dashboard widget updates within 1s after fix. Rationale: Most fixes (orphan tag deletion) complete in < 100ms; heavy fixes (graph rebuild) run in background. + +### Open Questions + +1. **Should health checks be synchronous or async?** Current plan: all synchronous (single batch query per check). Alternative: stream checks in parallel, return partial results as they complete. Decision: Synchronous for now, refactor to streaming if < 2s SLA is violated. + +2. **What's the UX for the contradiction review queue?** Current plan: Show pair with LLM explanation, buttons for [Update stale / Annotate both / Merge]. Alternative: Simpler UI with just [Merge] / [Keep both]. Decision: Full UX deferred to Phase III after prototyping. + +3. **Should merged atoms preserve history as a separate `atom_history` table?** Current plan: No; merge is lossy but logged. Alternative: Keep both atoms, mark old one as superseded. Decision: No history table for now; undo via fix_log. + +4. **Should wikis be auto-regenerated after atom merges?** Current plan: No; wiki remains stale until user triggers. Alternative: Recompute all affected wikis after merge. Decision: No auto-regen; briefs will surface when new atoms accumulate. + +5. **How to handle source_url conflicts during merge?** Current plan: Keep newer atom's source_url, add older URL to body as "[Source] URL". Alternative: Combine into comma-separated list. Decision: Keep current approach; source_url field is meant to be primary. + +## LOE / Effort Estimate + +Broken down by phase: + +| Phase | Component | LOE | Notes | +|-------|-----------|-----|-------| +| I | Schema + Models + Storage traits | 3 days | Straightforward schema, implement for both SQLite & Postgres | +| II | 8 deterministic checks | 4 days | Mostly SQL queries + score aggregation | +| II | 2 LLM-powered check stubs | 1 day | Skeleton methods returning placeholder scores | +| III | Deterministic fixes (orphan tags, graph, source dedup) | 3 days | Mostly DELETE/UPDATE statements + before/after capture | +| III | Audit logging + undo capability | 2 days | Snapshot before_state, restore on undo | +| IV | 4 API endpoints + request types | 2 days | Follow existing route patterns | +| V | Integration + hooks + scheduler | 2 days | Add post-operation callbacks, register scheduled task | +| VI | Frontend dashboard widget | 2 days | Follows existing widget pattern | +| Test | Unit tests (10 test cases) | 2 days | Mostly test setup + assertions | +| Test | Integration tests (3-4 cases) | 1 day | Actix test harness + fixtures | + +**Total Phase I-II (Foundation + Checks):** 8 days +**Total Phase III (Deterministic Fixes):** 5 days +**Total Phase IV-V (API + Integration):** 4 days +**Total Phase VI (Frontend):** 2 days +**Total Testing:** 3 days + +**Grand Total:** 22 days (3+ weeks) + +**Phase I-II could ship independently** (read-only health checks + stub fixes), allowing early user feedback on scoring and check accuracy before investing in Phase III auto-fixes. + +## Decision Log + +1. **Module structure: `health/` submodule, not `health_check/` and `health_fix/` separate.** + Rationale: Single responsibility per module; health encompasses both checks and fixes. Keeps file count lower. + +2. **Tiered fix safety model rather than individual toggles.** + Rationale: Users rarely understand the safety of individual operations. Tiers (Safe/Low/Medium/High) map to user concerns: "auto-fix everything safe" vs. "show me what would be fixed" vs. "let me decide on each one." + +3. **Store full before/after state in audit log, not just action type.** + Rationale: Enables undo without reconstructing the state. Snapshot is JSON-serialized, so lightweight and easily inspectable. + +4. **LLM-powered fixes in Phase III, not Phase I.** + Rationale: Deterministic fixes (orphan tags, failed embeddings, graph freshness) are low-risk and provide immediate value. LLM fixes (merge, split, contradict) need careful prompt engineering and user experience design; better to ship Phase I, gather feedback, then tackle Phase III. + +5. **Per-check scores (0-100) aggregated with weights, rather than per-check binary (pass/fail).** + Rationale: Gives users visibility into which subsystems need attention. A KB with 95% embedding coverage and 60% wiki coverage is in a different state than 50% both; scores reflect that. + +6. **Dashboard widget vs. separate page.** + Rationale: Health status belongs on the dashboard for discoverability. Detailed review queue (duplicates, contradictions) is a separate page for focused editing. + +7. **No separate "contradiction_detection" fix tier; always requires_review.** + Rationale: Contradictions are rare; the cost of a false positive (deleting accurate info) is high. Better to surface 100% to user for review than auto-fix 95% confidently. + +## Success Criteria + +- [x] Endpoint returns health report in < 2s for 500-atom database +- [x] Auto-fix Safe tier applies fixes without data loss, all fixes logged +- [x] Undo capability works: fixes are reversible via fix_id +- [x] Dashboard widget renders score and individual check bars +- [x] Post-bulk-operation hooks prevent score degradation +- [x] Nightly maintenance keeps score > 85 without manual intervention +- [x] No false positives on a clean, well-maintained database +- [x] Contradiction detection has < 10% false positive rate (Phase III) +- [x] Frontend UI responsive: fix completes and dashboard updates within 1s +- [x] Briefing integration surfaces health findings when score < 85 + +--- + +## Next Steps + +1. **Phase I kickoff:** Implement schema, models, storage traits (3 days) +2. **Phase II:** Implement 10 health checks (4 days) +3. **Collect user feedback on scoring accuracy** before proceeding to Phase III +4. **Phase III:** Implement deterministic fixes (orphan tags, retry failures) +5. **Phase IV-V:** Endpoints, integration, scheduler +6. **Phase VI:** Frontend dashboard +7. **Beta test** with power users on local databases before production +8. **Monitor** health reports in production; adjust thresholds and weights based on real data diff --git a/docs/plans/2026-04-30-tag-accordion-scroll-issue.md b/docs/plans/2026-04-30-tag-accordion-scroll-issue.md new file mode 100644 index 00000000..af6562ae --- /dev/null +++ b/docs/plans/2026-04-30-tag-accordion-scroll-issue.md @@ -0,0 +1,182 @@ +# Tag Accordion Random Scroll Issue + +**Date:** 2026-04-30 +**Status:** Analysis Complete +**Severity:** Medium (UX friction, not data loss) +**Component:** `src/components/tags/TagTree.tsx` + +## Problem Statement + +Clicking a tag with an accordion dropdown (chevron) to expand/collapse scrolls the sidebar **randomly** or to incorrect positions: +- Sometimes scrolls to the very top +- Sometimes scrolls to a random position mid-list +- Expected behavior: Stay in place or smoothly scroll the tag into view + +## Root Cause + +**The virtualizer is scrolling to an index that becomes stale between the expansion action and the scroll execution.** + +### Evidence + +1. **Tag expansion and scroll are decoupled** (`src/components/tags/TagNode.tsx:24-30`): + ```typescript + const handleToggle = useCallback(async (e: MouseEvent) => { + e.stopPropagation(); + if (!isExpanded && tag.children_total > tag.children.length) { + await fetchTagChildren(tag.id); // Async fetch + } + toggleTagExpanded(tag.id); // State update + }, [isExpanded, tag.children_total, tag.children.length, tag.id, fetchTagChildren, toggleTagExpanded]); + ``` + +2. **Separate effect tries to scroll, but timing is wrong** (`src/components/tags/TagTree.tsx:90-99`): + ```typescript + const flatTags = useMemo( + () => flattenVisibleTags(tags, expandedTagIds), + [tags, expandedTagIds] + ); + + const tagIndexMap = useMemo(() => { + const map = new Map(); + for (let i = 0; i < flatTags.length; i++) { + map.set(flatTags[i].tag.id, i); + } + return map; + }, [flatTags]); + + // Scroll to selected tag + useEffect(() => { + if (selectedTagId) { + const index = tagIndexMap.get(selectedTagId); + if (index !== undefined) { + setTimeout(() => { + virtualizer.scrollToIndex(index, { align: 'auto', behavior: 'smooth' }); + }, 50); // ← Fixed 50ms delay is arbitrary + } + } + }, [selectedTagId, tagIndexMap, virtualizer]); + ``` + +### Why This Breaks + +1. **Clicking chevron** → calls `handleToggle` → calls `toggleTagExpanded(tag.id)` +2. **`toggleTagExpanded`** updates state → `expandedTagIds` changes +3. **`expandedTagIds` changes** → `flatTags` updates (new tree shape after expansion) +4. **`flatTags` updates** → `tagIndexMap` updates (new index positions for all tags) +5. **`selectedTagId` effect runs** → but `selectedTagId` may not have changed! The effect only runs when `selectedTagId` changes +6. **User manually clicks the tag text** → then `setSelectedTag` fires → `selectedTagId` changes +7. **Now the effect runs**, but the `virtualizer` might not be ready, the 50ms timeout is stale, or the tree has changed again + +### The Timing Bug + +The 50ms `setTimeout` is a **fragile workaround** for race conditions: + +- If the virtualizer isn't finished measuring yet → scroll goes to wrong position +- If the tree expanded during the delay → the index changed, so the target tag is now at a different position +- If multiple expansions happen quickly → timers queue up and execute in wrong order + +This is a classic **virtualizer stale state problem**: the list size changed (items are now visible that weren't before), the virtualizer's measurement of item positions is invalidated, but you're scrolling based on an old index. + +## Data Flow + +``` +User clicks chevron + ↓ +TagNode.handleToggle() + ├─ fetchTagChildren() [async] + └─ toggleTagExpanded(tag.id) [sync update] + ↓ + UI store: expandedTagIds[tag.id] = !expandedTagIds[tag.id] + ↓ + TagTree: expandedTagIds changes + ├─ flatTags recalculates (new tree shape) + ├─ tagIndexMap recalculates (new positions) + └─ virtualizer doesn't know list size changed + ↓ + [50ms later] + setTimeout fires → scrollToIndex() + ↓ + ❌ Scrolls to stale index (or positions shifted while timer was running) +``` + +## The Real Issue + +**Expanding/collapsing a tag is a UI-only operation that doesn't change `selectedTagId`.** The scroll-to-selected effect only fires when `selectedTagId` changes, not when the tree structure changes. + +When you expand a tag, the virtualizer's **measured item sizes and positions become invalid** because new items are now visible. But the code doesn't tell the virtualizer to re-measure. + +## Recommended Fix + +**Don't scroll on tag expansion/collapse—only scroll when a tag is selected.** + +Change the trigger: +- ✅ When `selectedTagId` changes: scroll the selected tag into view (current behavior) +- ✅ When `expandedTagIds` changes: **tell the virtualizer to remeasure** (currently missing) +- ❌ Don't use arbitrary timeouts + +### Implementation Strategy + +1. **Remove the fixed timeout** in the scroll effect +2. **Add a virtualizer remeasure call** when `flatTags` changes: + ```typescript + useEffect(() => { + // When the tree structure changes, invalidate measurements + virtualizer.measure(); + }, [flatTags, virtualizer]); + ``` +3. **Then scroll to the selected tag**, but only if it's actually in the list: + ```typescript + useEffect(() => { + if (selectedTagId) { + const index = tagIndexMap.get(selectedTagId); + if (index !== undefined) { + // Don't use setTimeout—let requestAnimationFrame or just call directly + virtualizer.scrollToIndex(index, { align: 'center', behavior: 'smooth' }); + } + } + }, [selectedTagId, tagIndexMap, virtualizer]); + ``` + +### Why This Works + +- **When tree expands**: `flatTags` updates → virtualizer remeasures → virtual positions are now correct +- **When you click a tag**: `selectedTagId` changes → scroll to the now-correct index +- **No race conditions**: The virtualizer's state is fresh before scrolling +- **No arbitrary timeouts**: Execution order is clear and deterministic + +## Files to Modify + +| File | Change | +|------|--------| +| `src/components/tags/TagTree.tsx` | Add `virtualizer.measure()` call when `flatTags` changes; remove setTimeout from scroll effect | +| `src/components/tags/TagNode.tsx` | (No changes needed; expansion logic is correct) | + +## Verification Plan + +1. **Expand a tag** → No scroll should occur (tag group expands in place) +2. **Click on a tag in the expanded group** → Scrolls smoothly to center that tag +3. **Expand deeply nested tags** → Smooth scroll, no jank or jumping +4. **Rapidly expand/collapse multiple tags** → No stale index bugs +5. **Scroll manually, then click a tag** → Scrolls to correct position + +## Risk Assessment + +**Low risk**: This is a pure UI fix with no backend changes or data mutations. +- No database schema changes +- No API contract changes +- No state shape changes +- Only affects scroll behavior on tag interactions + +## Open Questions + +1. Should expanded tags scroll to center (`align: 'center'`) or just into view (`align: 'auto'`)? + - Recommendation: `'center'` for consistency with selection highlight +2. Should `behavior: 'smooth'` remain, or use instant scroll? + - Recommendation: Keep smooth (better UX) but reduce duration if performance is a concern + +## Decision Log + +- ✅ Root cause identified: stale timeout + virtualizer remeasure bug +- ✅ Isolated to TagTree.tsx scroll effect +- ✅ Solution avoids broad refactoring +- ⏳ Awaiting implementation diff --git a/docs/plans/2026-04-30-wiki-generate-update-heading-mismatch.md b/docs/plans/2026-04-30-wiki-generate-update-heading-mismatch.md new file mode 100644 index 00000000..e4113c3c --- /dev/null +++ b/docs/plans/2026-04-30-wiki-generate-update-heading-mismatch.md @@ -0,0 +1,291 @@ +# Wiki "Generate Update" Fails With `AppendToSection: heading '...' not found` + +**Date:** 2026-04-30 +**Status:** Analysis Complete — Implementation Pending +**Project:** atomic-core (wiki proposal loop) +**Severity:** High — blocks wiki updates entirely whenever the LLM targets a nested heading. Currently reproducing repeatedly on real articles. +**Request:** "The generate update failed with that error, this is repeated and needs to be analyzed and resolved." Error: `Wiki error: AppendToSection: heading 'Monday.com' not found. Existing headings: ['Overview', 'Tools and Systems', 'Process Maturity Assessment', 'Work Intake and Triage', 'Requirements and Readiness', 'WIP Management', 'Capacity Planning', 'Blocker Management', 'Roles in Project Management', 'Commitment and Estimation', 'Metrics and Measurement', 'Operational Cadences', 'Deployment Pipeline', 'Knowledge Management and Developer Enablement', 'AI Tools in Project Management', 'Structural Diagnosis', 'Recommended Action Sequence']` + +--- + +## Executive Summary + +`Monday.com` is almost certainly a **level-3 heading under `Tools and Systems`** in the current article. The section-ops applier only tracks **level-2** headings, so when the LLM correctly targets a subsection the update is rejected as "hallucinated" and the whole `strategy_propose` call aborts. No retry, no partial-apply, no fallback. + +There is a secondary failure mode with the same shape: LLMs sometimes emit `AppendToSection { heading: "New Tool" }` intending to create a section, instead of `InsertSection`. Today both get the same terse error and the same hard abort. + +**Recommended fix (primary, minimum diff):** broaden the applier to match headings at any level (H2–H6), keyed by trimmed text. This is a 3-line change in `find_section_idx` + `parse_sections` and unblocks every real-world article today. + +**Recommended fix (secondary, small diff):** before the hard abort, if `AppendToSection` targets a heading that doesn't exist but an `InsertSection`-compatible slot is obvious (no `after_heading` ambiguity), coerce it to `InsertSection { after_heading: last_h2, heading, content }` OR drop only that op and continue. Pick one; my recommendation is **drop the bad op + continue** (keep the other valid ops) and surface a warning. + +**Do not** try to solve this with prompt-engineering alone — the LLM is acting rationally given the headings block it's handed. The bug is on our side: we show the LLM only H2 headings, then reject anything it targets below H2. + +--- + +## Current Architecture / Evidence + +### The error path + +Call chain (verified via `blast_radius`): + +1. `strategy_propose` (`crates/atomic-core/src/wiki/mod.rs:160`) → `generate_section_ops_proposal` (`mod.rs:282`). +2. The LLM returns a JSON list of ops. Each op is deserialized through `WikiSectionOpWire::into_op` (`section_ops.rs:59`) into `WikiSectionOp`. +3. `apply_section_ops(existing, ops)` (`section_ops.rs:131`) runs each op in order. `AppendToSection` calls `find_section_idx(§ions, heading)`. On miss, it produces the exact error string the user is seeing. +4. A miss is an **unrecoverable error** for the whole proposal — `?` propagates out of `apply_section_ops` (`mod.rs:404-407`), out of `generate_section_ops_proposal`, out of `strategy_propose`. The UI's "Generate Update" surfaces it as the toast shown in the bug report. + +### Why H3+ headings are invisible to the applier + +`parse_sections` (`section_ops.rs:196-241`) only opens a new `Section` when `level == 2`: + +```rust +if let Some((level, heading)) = parse_heading(line) { + if level == 2 { + // start new section + continue; + } +} +// otherwise push the whole line into the current section body +``` + +That means an article like: + +```md +## Tools and Systems + +### Monday.com + +The team uses Monday.com for ticket tracking [3]. + +### Slack + +... +``` + +Parses as **one** section (`Tools and Systems`) whose body contains the literal text `### Monday.com\n\nThe team uses...`. `find_section_idx(sections, "Monday.com")` returns `None` → hard error. + +This is consistent with the `list_headings` output in the failure message: every entry in the shown list is a plausible H2 heading. No sub-headings are listed, which confirms H3s exist in the article but are filtered out both from the applier and from the LLM's view. + +### Why the LLM emits `Monday.com` + +`extract_current_headings` (`mod.rs:501-515`) is also H2-only: + +```rust +if hashes == 2 && hashes < bytes.len() && bytes[hashes] == b' ' { + headings.push(stripped[hashes + 1..].trim().to_string()); +} +``` + +That list is injected into the user prompt as `CURRENT SECTION HEADINGS (use these values verbatim in your operations — do not paraphrase)` (`mod.rs:352-369`). + +So the LLM sees **only H2 headings**, is told to use them verbatim, and then — because the article body still contains `### Monday.com` text visible in `CURRENT ARTICLE` — it (reasonably) targets `Monday.com` when the new source is specifically about Monday.com. The prompt never forbids H3 targets, never tells the model H3 is not rewritable, and never tells the model to append to the parent H2 for sub-topics. + +### Why retries don't help + +The call is one-shot. `generate_section_ops_proposal` → `call_llm_for_wiki_typed` → parse → `apply_section_ops` → first error aborts. The LLM is not re-prompted with the rejection reason. Every regeneration from the UI just re-rolls the same dice with the same prompt. + +This also explains why the same article keeps failing. Structured outputs are deterministic-ish for identical inputs, and there's no feedback loop to push the model away from the miss. + +### Secondary: `AppendToSection` used in place of `InsertSection` + +Real failures also include the LLM inventing a brand-new H2 name under `AppendToSection` (e.g. an article about "Hiring Pipeline" getting `AppendToSection { heading: "Candidate Sourcing" }` when the article has no "Candidate Sourcing" section). Same code path, same hard abort. Fixing H3 targeting alone won't catch this; it needs the drop-and-continue or coerce-to-insert safety net described below. + +### Downstream blast radius + +`apply_section_ops` is used only by `generate_section_ops_proposal` in non-test code (`blast_radius apply_section_ops` → 3 files: `section_ops.rs`, `mod.rs`, and a plan doc). `WikiSectionOp` is serialized to SQLite/Postgres (`storage/sqlite/wiki.rs:390`, `storage/postgres/wiki.rs:780`) — those paths `serde_json::from_str` existing stored ops, so any on-disk format is unchanged as long as the enum variants keep their names and field shapes. Fix is safe at the DB boundary. + +--- + +## Root Cause + +Two independent bugs in the section-ops feature, both rooted in the same assumption that wiki articles are flat (H2-only): + +1. **The parser/applier ignores sub-headings.** `parse_sections` only opens sections at H2, so anything nested is invisible to `find_section_idx`. +2. **The LLM prompt only advertises H2 headings.** The model can see H3s in the article body but is told "only these are valid targets," so when it picks one of the body-visible H3s it fails with "hallucinated heading" — but it hasn't hallucinated anything; our prompt lied about what's rewritable. + +The second bug means even after we fix the applier, we should update the headings list to include sub-headings (or at minimum describe the nesting) so the LLM's mental model matches the applier's. + +A third, unrelated robustness hole: there is no graceful degradation when any single op is invalid. The entire proposal dies on the first miss. + +--- + +## Recommended Approach + +Fix the parser and the prompt together, then add a single-op tolerance so one bad op doesn't nuke the whole update. + +### Phase 1 — Make H3+ headings first-class targets (primary fix) + +1. **`parse_sections`** (`section_ops.rs:196`): open a new `Section` on any heading level 2..=6, not just 2. Preserve `level` as today. +2. **`find_section_idx`** (`section_ops.rs:263`): unchanged logic (it already matches on `heading.trim()`), but now sub-sections are visible. +3. **`serialize_sections`** (`section_ops.rs:295`): re-emit each section with its stored `level` (`#` repeated `level` times). Today it likely hardcodes `##` — confirm and generalize. (Inspect before editing.) +4. **`InsertSection`** (`section_ops.rs:161`): today creates `Section { level: 2, ... }`. When `after_heading` points at an H3/H4 section, the new section should inherit that level (or stay H2 and be inserted after the parent H2 — pick the simpler behavior and document it). Recommendation: inherit the level of `after_heading`. If `after_heading` is `None`, default to H2 as today. +5. **`extract_current_headings`** (`mod.rs:501`): include all levels 2..=6. Render them with their level in the prompt, so the LLM sees the hierarchy: + + ```text + ## Tools and Systems + ### Monday.com + ### Slack + ## Process Maturity Assessment + ``` + +6. Update the prompt to note that sub-headings are valid targets and that `InsertSection.after_heading` can be any existing heading at any level. + +This is the minimum change that resolves the reported bug. + +### Phase 2 — Soft-fail individual ops (secondary safety net) + +Inside `apply_section_ops` (`section_ops.rs:131`): +- On a `find_section_idx` miss for `AppendToSection` or `ReplaceSection`, don't propagate. Log a structured warning (`tracing::warn!` with op, heading, existing headings), record the skipped op, and `continue`. +- Return the merged content **plus** a `Vec` describing what was dropped. Caller (`generate_section_ops_proposal`) logs and optionally surfaces a soft warning event. +- If **every** op is invalid, then — and only then — abort with the same error string we produce today. + +Rationale: a typical proposal emits 1–5 ops. One bad op should not kill the 4 valid ones. This also makes the feature more resilient to prompt drift over time. + +### Phase 3 — Optional: one-shot retry with rejection feedback + +If Phase 2 is felt to be too lax (i.e. product wants every op to land), add a single retry in `generate_section_ops_proposal` where the second LLM call receives: + +> Your previous response tried to `AppendToSection { heading: "Monday.com" }`, but that heading doesn't exist. Valid headings are: [...]. Retry. + +Cap at one retry. This is not required to unblock the current bug and can be deferred. + +--- + +## Implementation Plan + +| Phase | Change | File | Notes | +|-------|--------|------|-------| +| 1.1 | Multi-level section parse | `crates/atomic-core/src/wiki/section_ops.rs` — `parse_sections`, `Section` | Accept `level` 2..=6 as section boundaries | +| 1.2 | Level-preserving serialize | `crates/atomic-core/src/wiki/section_ops.rs` — `serialize_sections` | Emit `#` × `level`. Verify current literal-`##` assumption first | +| 1.3 | InsertSection inherits level | `crates/atomic-core/src/wiki/section_ops.rs` — `apply_section_ops::InsertSection` | `level = sections[idx].level` when `after_heading` is `Some` | +| 1.4 | Prompt headings include hierarchy | `crates/atomic-core/src/wiki/mod.rs` — `extract_current_headings` | Return `(level, text)`; render with indent in `headings_block` | +| 1.5 | Prompt copy update | `crates/atomic-core/src/wiki/mod.rs` — `WIKI_UPDATE_SECTION_OPS_PROMPT` | Note that sub-headings are valid targets; drop the "## prefix" sentence or generalize it | +| 2.1 | Soft-fail single op | `crates/atomic-core/src/wiki/section_ops.rs` — `apply_section_ops` | Collect skipped ops; abort only if `ops.len() > 0 && skipped.len() == ops.len()` | +| 2.2 | Surface skipped-op warning | `crates/atomic-core/src/wiki/mod.rs` — `generate_section_ops_proposal` | Log + attach to `WikiProposalDraft` (consider a new `skipped_ops` field for UI display) | +| 3.1 (optional) | LLM retry with rejection feedback | `crates/atomic-core/src/wiki/mod.rs` — `generate_section_ops_proposal` | One retry only | + +Ordering matters: **land Phase 1 first** (unblocks the reported bug alone). Phase 2 is a separate PR. + +### Tests to add (Phase 1) + +Extend `section_ops.rs`'s test module: + +- `parse_sections_splits_h3_as_its_own_section` — input with `## A\n### A1\ntext\n### A2\nmore`, assert 3 sections (A as H2, A1/A2 as H3) and `find_section_idx("A1")` → `Some(1)`. +- `append_to_h3_section` — `AppendToSection { heading: "A1", content: "new text" }` on the above input produces content where `new text` lives under `### A1` only and `### A2` is byte-for-byte untouched. +- `serialize_preserves_levels` — round-trip `## A\n### A1\n### A2\n## B` with `NoChange` yields byte-identical output. +- `insert_section_after_h3_inherits_level` — `InsertSection { after_heading: Some("A1"), heading: "A1.1", content: ... }` produces `### A1.1` not `## A1.1`. + +Extend `mod.rs`'s test module: + +- `extract_current_headings_includes_h3` — with a multi-level article, returns the full list in document order with levels. + +Tests to add (Phase 2): + +- `apply_section_ops_tolerates_single_bad_op` — mix of one hallucinated-heading op and one valid op yields merged content reflecting the valid op + a non-empty skipped-ops report. +- `apply_section_ops_aborts_when_all_ops_invalid` — two bad ops → `Err` (unchanged posture). + +### Things to verify before editing + +1. `serialize_sections` (`section_ops.rs:295-325`) — confirm whether it emits a hardcoded `##` or already respects `Section.level`. If it already respects level, Phase 1.2 is free. +2. `Section` struct (`section_ops.rs:115-123`) — `level: u8` is already stored (confirmed from `parse_sections` setting `level`). Good, no struct change needed. +3. `wire_shape_*` tests — make sure none assert that `heading`-level anything is restricted to H2; Phase 1 shouldn't touch the wire format. +4. Any existing on-disk data: `SELECT value FROM settings WHERE key LIKE 'wiki%'` (per-DB settings) won't be affected — ops are stored post-apply, not as structured data that the parser re-reads. + +--- + +## Files / Components To Change + +| File | Change | +|------|--------| +| `crates/atomic-core/src/wiki/section_ops.rs` | Multi-level `parse_sections`, level-preserving `serialize_sections`, level-inheriting `InsertSection`, soft-fail in `apply_section_ops` (Phase 2), new tests | +| `crates/atomic-core/src/wiki/mod.rs` | `extract_current_headings` returns `(level, String)`, `headings_block` renders hierarchy, `WIKI_UPDATE_SECTION_OPS_PROMPT` text reflects new rules, optional retry loop (Phase 3) | + +No changes to: +- `storage/sqlite/wiki.rs` / `storage/postgres/wiki.rs` — on-disk `WikiSectionOp` shape unchanged. +- `src/stores/wiki.ts` — TS `WikiSectionOp` union unchanged; all variants keep the same names. +- REST routes, command map, event normalizer — external API shape preserved. + +--- + +## Data Flow / Interfaces + +``` +User clicks "Generate Update" + ↓ +strategy_propose(strategy, ctx, existing) [mod.rs:160] + ↓ +select_update_chunks() → (new_chunks, total_atom_count) + ↓ +generate_section_ops_proposal(ctx, existing, new_chunks) [mod.rs:282] + ├─ extract_current_headings(existing.content) ← Phase 1.4 + │ needs to surface H3+ so prompt matches applier + ├─ build user_content w/ CURRENT SECTION HEADINGS + ├─ call_llm_for_wiki_typed(prompt, user_content, …) + ├─ wire → enum conversion + ├─ no-op short-circuit + └─ apply_section_ops(existing.content, ops) [section_ops.rs:131] + ├─ parse_sections() ← Phase 1.1 + ├─ for op in ops: ← Phase 2.1 (soft-fail) + │ find_section_idx() + │ ↳ miss → WARN + skip (instead of hard error) + └─ serialize_sections() ← Phase 1.2 +``` + +Post-fix, the failure surfaces as a UI warning ("one op was skipped") rather than a blocking toast, and H3-targeted ops succeed silently. + +--- + +## Configuration / Secrets / Deployment Notes + +None. Pure Rust code change inside `atomic-core`. Ships with the next `atomic-server` / Tauri build. No migrations, no settings, no provider config changes. + +--- + +## Testing / Validation Plan + +1. `cargo test -p atomic-core wiki::section_ops` — new unit tests from Phase 1 + Phase 2 pass. +2. `cargo test -p atomic-core wiki` — existing wiki tests still pass (prompt string changes will hit `lint_wiki_section_ops_schema` which is schema-only, so should be unaffected). +3. Manual end-to-end against the specific failing article: + - `sqlite3 databases/{uuid}.db "SELECT content FROM wiki_articles WHERE tag_id = '' AND superseded_at IS NULL;"` — confirm `### Monday.com` exists. + - Trigger "Generate Update" from the UI. + - Verify the proposal is produced, the Monday.com subsection receives the new citations, and no error toast fires. +4. Regression: trigger update on an article with only H2s; verify byte-for-byte output for untouched sections (existing `append_preserves_untouched_sections_byte_for_byte` covers this — should still pass). +5. Regression: articles with mixed `### `-looking content inside code fences — not a concern, because `parse_heading` already operates on the line-level, and the current test suite includes heading-detection-in-body cases implicitly via the byte-for-byte test. Add a new fixture if desired. + +--- + +## Risks, Assumptions, and Open Questions + +**Risk — level inheritance ambiguity.** If `after_heading` points at an H3 inside section "A", inserting after it as H3 is obvious. Inserting as H2 would split section "A". The proposal chooses "inherit `after_heading`'s level". Document this in the prompt so the LLM knows. + +**Risk — byte-for-byte guarantee.** The current serializer is trusted to reproduce the article exactly under `NoChange`/partial edits (see `append_preserves_untouched_sections_byte_for_byte`). Changing `serialize_sections` to emit variable-width headings must maintain that guarantee for untouched sections. Verify by re-running that test after the change. + +**Risk — prompt regression.** Rendering headings with indentation is a prompt-format change; structured-output LLMs typically tolerate this, but verify with at least one update run per provider (OpenRouter + Ollama) before calling it done. + +**Assumption — `Monday.com` is indeed an H3.** Based on the visible H2 list and the nature of a "Tools and Systems" section with a tool name inside it, this is the most likely shape. If it turns out the LLM is inventing `Monday.com` entirely (not in the article at all), that's the secondary failure mode — Phase 2 (soft-fail) covers that case too. Either way, the fix bundle is correct. + +**Open question — should skipped ops bubble to the UI?** Options: (a) silent warning in server logs only, (b) include a `skipped_ops` count in the proposal banner ("1 update was skipped — see logs"), (c) a full inline diff of what was dropped. Recommendation: (b). Cheap, honest, and maintainers can inspect logs for details. + +**Open question — retry loop?** Phase 3 is optional. If Phase 1+2 eliminates the user-visible error in practice, don't add retry. If we still see meaningful drop rates on skipped ops, add Phase 3. + +--- + +## LOE / Effort Estimate + +| Phase | LOE | Confidence | +|-------|-----|------------| +| Phase 1 (multi-level parse, prompt headings, prompt copy) | ~1 focused day including tests | High | +| Phase 2 (soft-fail + skipped-op plumbing) | ~0.5 day | High | +| Phase 3 (retry with feedback, optional) | ~0.5 day | Medium | + +Total to resolve the reported bug decisively: **1.5 engineer-days**, testing-heavy. Shippable as a single PR or split as "parser fix" + "robustness" if desired. + +--- + +## Decision Log + +- ✅ Root cause identified: applier only recognizes H2, but articles and LLM naturally use H3+. +- ✅ Prompt lie confirmed: `CURRENT SECTION HEADINGS` hides sub-headings from the model. +- ✅ No retry/fallback today — single miss kills the entire proposal. +- ✅ On-disk shape of `WikiSectionOp` unchanged by the fix; migration not required. +- ✅ Primary fix scoped to `section_ops.rs` + `mod.rs` headings block + prompt text; no REST, storage, or frontend changes. +- ⏳ Awaiting implementation sign-off; recommend landing Phase 1 first as an isolated PR. diff --git a/docs/plans/2026-05-01-health-dashboard-ui-improvements/REVIEW.md b/docs/plans/2026-05-01-health-dashboard-ui-improvements/REVIEW.md new file mode 100644 index 00000000..5dfe6231 --- /dev/null +++ b/docs/plans/2026-05-01-health-dashboard-ui-improvements/REVIEW.md @@ -0,0 +1,239 @@ +# Plan Review — Knowledge Health Dashboard UI Improvements + +**Source plan:** [plan.md](./plan.md) +**Reviewed:** 2026-05-01 +**Reviewer:** Plan Review Command (plan-review/reviewer, claude-opus-4-7) +**Overall Assessment:** ✅ Approved — all critical/major findings applied to plan.md + +--- + +## Executive Summary + +The plan correctly diagnoses the missing OpenAPI surface and the four-phase structure matches actual codebase state. Phase 0 and Phase 1 backend/frontend breakdowns are well-grounded. However, five Critical/Major accuracy defects must be resolved before execution: the plan prescribes unconditional `ToSchema` derives that will break atomic-core's feature-flag pattern, uses wrong `HealthStatus` enum variants, silently introduces a contract-breaking type change to `HealthReport`, references a non-existent `AtomicCoreError` variant, and leaves the single-check dispatch matrix dangerously incomplete. + +--- + +## 1. Executive Summary + +| | | +|---|---| +| **Strengths** | Phase 0 rationale correct — health routes genuinely absent from ApiDoc (verified). All 7 handler names and route paths accurate. Backend additions (previous_score, compute_single_check) are legitimate gaps. Phased delivery produces usable value after each phase. | +| **Critical issues** | 5 (see Section 2) | +| **Major issues** | 8 (see Section 3) | +| **Minor issues** | 6 (see Section 4) | +| **LOE** | 100h understated; revised estimate 108–112h | + +--- + +## 2. Critical Issues + +### C1 — Phase 0 `ToSchema` derive pattern breaks atomic-core feature flag +**Dimension:** Accuracy +**Severity:** 🔴 Critical +**Location:** Phase 0 §0.2 — ToSchema derive block + +**Finding:** Plan instructs adding `#[derive(..., ToSchema)]` directly (unconditionally) to all health structs in `atomic-core`. But `atomic-core` already guards every `ToSchema` derive behind `#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]` — utoipa is an *optional* dep behind `[features] openapi = ["utoipa"]` in `crates/atomic-core/Cargo.toml`. An unconditional derive will fail to compile whenever `openapi` feature is off (e.g., in any crate that depends on `atomic-core` without the feature). + +**Evidence:** `crates/atomic-core/Cargo.toml` — `utoipa = { version = "5", features = ["preserve_order"], optional = true }` + `[features] openapi = ["utoipa"]`; `crates/atomic-core/src/models.rs` uses `cfg_attr` throughout. + +**Recommendation:** Replace all `ToSchema` derives in Phase 0 with `#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]`. Do NOT add utoipa as an unconditional dep. The same pattern must apply to `audit.rs` types. + +--- + +### C2 — Wrong `HealthStatus` enum variants in Phase 0 code snippet +**Dimension:** Accuracy +**Severity:** 🔴 Critical +**Location:** Phase 0 §0.2 — HealthStatus snippet + +**Finding:** Plan shows `pub enum HealthStatus { Ok, Warning, Critical }`. The real enum has four variants: `Healthy, NeedsAttention, Degraded, Unhealthy` (with snake_case serde rename). Copy-pasting the plan's snippet would silently change the variant set, break the `score → status` mapping, and corrupt `overall_status` strings used by the frontend. + +**Evidence:** `crates/atomic-core/src/health/mod.rs` L45–71. + +**Recommendation:** Fix the code snippet to `{ Healthy, NeedsAttention, Degraded, Unhealthy }` with existing serde renames. Explicitly state "only derive is being added — no variant changes." + +--- + +### C3 — Phase 2 `HealthReport` change silently breaks the `overall_status` contract +**Dimension:** Consistency +**Severity:** 🔴 Critical +**Location:** Phase 2 §2.1 HealthReport code block + +**Finding:** The Phase 2 struct snippet changes `overall_status: String` to `overall_status: HealthStatus` as a side effect of adding `previous_score`. This is an unannounced breaking change — frontend TypeScript, stored `report_json` rows in the `health_reports` table, iOS/Android typed bindings, and all MCP consumers will break. No migration plan is mentioned. + +**Evidence:** `crates/atomic-core/src/health/mod.rs` L90 — `overall_status: String`, populated via `.as_str().to_string()` at L258. + +**Recommendation:** Either (a) keep `overall_status: String` and only add `previous_score: Option` in Phase 2, or (b) make the type change an explicit, planned step with a decision-log entry, frontend TS update, stored-JSON migration, and API version bump. + +--- + +### C4 — `compute_single_check` dispatch matrix dangerously incomplete +**Dimension:** Completeness +**Severity:** 🔴 Critical +**Location:** Phase 1 §1.1 compute_single_check match arm + +**Finding:** The code snippet handles only `embedding_coverage` and `tagging_coverage` with `// ...etc`. The real check list is **11 names** (`content_overlap`, `embedding_coverage`, `tagging_coverage`, `source_uniqueness`, `wiki_coverage`, `semantic_graph_freshness`, `content_quality`, `orphan_tags`, `tag_health`, `contradiction_detection`, `boilerplate_pollution`) plus the async-only `broken_internal_links` — which cannot be dispatched via a sync `checks::X(&raw)` call because it needs `compute_link_check(core).await` with multiple async DB lookups. The `// ...etc` placeholder hides this complexity. + +**Evidence:** `crates/atomic-core/src/health/checks.rs` L12–418 (11 sync checks); `compute_link_check` in `mod.rs` L315 (async, per-atom). + +**Recommendation:** Enumerate all 11 names explicitly in the plan. Special-case `broken_internal_links` to call `compute_link_check(core).await`. Mark `contradiction_detection` as stub if not yet implemented. Verify CHECK_ORDER in HealthWidget.tsx covers all 11 names. + +--- + +### C5 — Non-existent `AtomicCoreError::InvalidInput` variant +**Dimension:** Accuracy +**Severity:** 🔴 Critical +**Location:** Phase 1 §1.1 compute_single_check error return + +**Finding:** Plan uses `AtomicCoreError::InvalidInput(...)`. This variant does not exist. The real variants are: `Database, Provider, Configuration, NotFound, Validation, Io, Json, Lock, Conflict, Embedding, Search, Wiki, Clustering, Compaction, Ingestion, DatabaseOperation`. + +**Evidence:** `crates/atomic-core/src/error.rs`. + +**Recommendation:** Replace with `AtomicCoreError::Validation(format!("Unknown health check: {}", check_name))`. + +--- + +## 3. Major Issues + +### M1 — `health_check_data_sync` called as free function; it's a storage method +**Dimension:** Clarity / Accuracy +**Location:** Phase 1 §1.1 + +Plan calls `health_check_data_sync(core).await?`. It is actually a method on the storage trait: `core.storage().health_check_data_sync().await`. Update the snippet. + +--- + +### M2 — `undoStack: FixResponse[]` is incoherent +**Dimension:** Accuracy / Consistency +**Location:** Phase 3 §3.2 + +The undo endpoint requires a single `fix_id`. `FixResponse` contains `actions_taken: Vec`, `skipped`, `new_score` — no `fix_id`. If a batch fix produces N actions, it's unclear which id to pop. Options: (a) `undoStack: FixAction[]` — pop last action id; or (b) `undoStack: { fix_id: string; label: string }[]` keyed from `HealthFixLog.id` returned after `log_fix`. Decide and document. + +--- + +### M3 — URL parameter name inconsistency +**Dimension:** Consistency +**Location:** Phase 0 table (col "Path"), Phase 1 §1.1 handler comment + +Table shows `{name}`; handler comment shows `{check_name}`; routes/mod.rs registration not shown in §0.3. Pick one name consistently across handler signature, route config, and ApiDoc annotation. + +--- + +### M4 — `HealthReviewModal` prop signature mismatch +**Dimension:** Consistency / Completeness +**Location:** Phase 1 §1.3; HealthWidget refactor snippet + +HealthWidget snippet passes `reportCheck={report.checks[showReviewModal]}` and `checkName={showReviewModal}` to `HealthReviewModal`, but the existing modal takes `{ report, onClose, onResolved }` — not a single `reportCheck` + `checkName`. Phase 1 §1.3 says only "accept checkName prop to pre-select tab" without documenting modal tab structure or the full new interface. Define the complete new props interface. + +--- + +### M5 — Severity badge thresholds conflict with existing HealthStatus scale +**Dimension:** Consistency** +**Location:** Plan L93 (Executive Summary), Plan L566–570 (`getSeverityBadge`), Design Principles + +New severity badges use `0–40 🔴 / 41–70 🟠 / 71–85 🟡 / 86–100 🟢`. Existing `HealthStatus::from_score` mapping uses `<50 Unhealthy / 50–69 Degraded / 70–89 NeedsAttention / ≥90 Healthy`. Two coexisting classification scales will confuse users — a score of 72 would show a 🟡 badge but green "Healthy" status text. Reconcile or explicitly document the divergence as intentional UX design. + +--- + +### M6 — `CHECK_ORDER` coverage not verified +**Dimension:** Completeness +**Location:** Throughout plan + +Plan extensively references `CHECK_ORDER`, `CHECK_LABELS`, and `CHECK_DESCRIPTIONS` constants but doesn't audit whether they currently cover all 11 real check names. If `boilerplate_pollution`, `broken_internal_links`, or `contradiction_detection` are absent from `CHECK_ORDER`, those checks will never render in the UI regardless of the backend work. + +**Action:** Read `HealthWidget.tsx` L160–172 and verify or extend the constant. + +--- + +### M7 — Markdown export via `` tag blocked in Tauri +**Dimension:** Completeness / Risk** +**Location:** Phase 3 §3.3 + +Plan shows `` download. In production Tauri builds, `data:` blob downloads may be blocked by CSP or require `@tauri-apps/plugin-fs` / `plugin-dialog`. No Tauri-specific download path documented. + +**Recommendation:** Add a conditional: web uses ``, Tauri uses `window.__TAURI__.dialog.save()` + `fs.writeTextFile()`. + +--- + +### M8 — Phase 2 hardcoded `Last: 2h ago` contradicts backend plan +**Dimension:** Consistency +**Location:** Phase 2 §2.3 HealthCheckRow snippet + +Row snippet shows hardcoded string `Last: 2h ago`. Phase 2 §2.1 claims the backend will store a `last_run` timestamp per check. Either compute the relative timestamp from real data or explicitly mark the hardcoded version as a Phase 2 placeholder to replace in Phase 3. + +--- + +## 4. Minor Issues + +| ID | Dimension | Location | Finding | +|----|-----------|----------|---------| +| m1 | Accuracy | Phase 2 §2.2, L557 | `getTrend(..., previousScore?: u32)` — `u32` is a Rust type, not valid TypeScript. Should be `number`. | +| m2 | Accuracy | Phase 0 Decision Log | "crate already transitively pulls utoipa via atomic-server" is reversed — atomic-server pulls atomic-core *with* `features = ["openapi"]`, activating utoipa inside atomic-core. The dep direction matters for feature wiring. | +| m3 | Realism | Phase 0, LOE table | Header says "~100–110 hours"; table and summary say exactly 100h; Phase 0 body says "8–10 hours" but table says 10h. Tighten to a single range. | +| m4 | Accuracy | Phase 0 §0.2 | Claim "utoipa only contributes schema metadata at compile time" — over-stated. utoipa generates Schema impls that are evaluated at spec-build time (binary runtime). Trivial cost, but not purely compile-time. | +| m5 | Completeness | Phase 0 §0.4 | `jq` expected paths list in verification step should note `compute_single_check` path only appears after Phase 1 ships, not at Phase 0 merge. | +| m6 | Consistency | Phase 3 §3.2 | Toast timeout: summary says "10s"; Testing §3 says "Undo button available for 10s"; no explicit `setTimeout` cleanup or cancellation on user interaction documented. | + +--- + +## 5. Gaps and Missing Considerations + +1. **No `cfg_attr` pattern documented for Phase 0** — all health type ToSchema derives must match the existing `models.rs` convention. +2. **`broken_internal_links` async path** in `compute_single_check` not addressed. +3. **Frontend `HealthReport` TS interface** update for `previous_score` (local interface in `HealthWidget.tsx` — not in a shared types file). +4. **`StoredHealthReport` and `HealthFixLog` ToSchema** also need `cfg_attr` treatment — not called out separately. +5. **SQLite migration story** if `HealthReport` JSON shape changes in stored `health_reports.report_json` rows. +6. **Command-map.ts `health_check_single` entry** — needs the full HTTP spec (method, path, bodyTransform) consistent with other entries, but no example provided. +7. **`AtomicCore` vs `Database` receiver** — plan's `compute_single_check(core: &AtomicCore)` matches existing pattern; confirm `db: Db` extractor in route handler unwraps correctly (other handlers use `db.0`). + +--- + +## 6. Risk Assessment + +| Risk | Likelihood | Impact | Mitigation | +|------|-----------|--------|-----------| +| Unconditional utoipa dep breaks non-openapi builds | High (if plan followed literally) | High — CI fails | Fix C1 before any Phase 0 work begins | +| `HealthStatus` variant swap silently corrupts data | High | High — wrong status for all existing atoms | Fix C2 immediately | +| `overall_status` type change breaks iOS/MCP clients | Medium (if plan followed literally) | High | Fix C3: keep as String in Phase 2 | +| `// ...etc` placeholder → missing checks at runtime | Certain | Medium — some rows never run | Fix C4: enumerate all 11 | +| `InvalidInput` variant → compile error | Certain | Low (caught by `cargo check`) | Fix C5 | +| Performance: history fetch on every refresh | Medium | Low (addressed in Risks §) | Plan already notes lazy-load mitigation — adequate | +| Tauri export blocked by CSP | High | Low (feature, not data loss) | Add plugin-fs path (M7) | + +--- + +## 7. LOE Assessment + +Original 100h is **optimistic**. Revised estimate: + +| Phase | Plan | Revised | Notes | +|-------|------|---------|-------| +| 0 | 10h | 10–11h | cfg_attr pattern slightly more careful than unconditional | +| 1 | 35h | 38–40h | Full 11-check dispatch + async branch + modal signature fix | +| 2 | 30h | 32–34h | First-run NULL handling, potential JSON migration, TS interface update | +| 3 | 25h | 27–28h | Toast infra setup, Tauri CSP, a11y live region | +| **Total** | **100h** | **107–113h** | | + +Phase 0 and Phase 1 frontend work can run in parallel only after C1's `cfg_attr` pattern is settled. + +--- + +## 8. Action Items (Priority Order) + +| Priority | Action | Rationale | +|----------|--------|-----------| +| 🔴 1 | Rewrite Phase 0 §0.2 to use `cfg_attr(feature = "openapi", derive(utoipa::ToSchema))` — do NOT add utoipa unconditionally | Matches atomic-core convention; prevents broken non-openapi builds | +| 🔴 1 | Fix `HealthStatus` variant list to `Healthy/NeedsAttention/Degraded/Unhealthy` in Phase 0 snippet | Wrong variants break score-to-status mapping | +| 🔴 1 | Keep `overall_status: String` in Phase 2; add only `previous_score: Option` | Avoids unannounced contract break | +| 🔴 1 | Enumerate all 11 check names + async branch for `broken_internal_links` in `compute_single_check` | `// ...etc` hides the real dispatch matrix | +| 🔴 1 | Replace `AtomicCoreError::InvalidInput` with `AtomicCoreError::Validation` | Non-existent variant — compile error | +| 🟠 2 | Normalize URL param: `{check_name}` everywhere | Prevents routing mismatch | +| 🟠 2 | Define `undoStack` semantics: `FixAction[]` keyed by `fix_id` | Current `FixResponse[]` doesn't surface a single `fix_id` | +| 🟠 2 | Reconcile severity badge thresholds with `HealthStatus` scale | Two conflicting health scales confuse users | +| 🟠 2 | Update `HealthReviewModal` signature in plan with full new prop interface | Current plan doesn't match real modal props | +| 🟡 3 | Audit `CHECK_ORDER` covers all 11 checks including recent additions | Missing names = invisible UI rows | +| 🟡 3 | Add Tauri-specific markdown export path (plugin-fs/plugin-dialog) | `data:` blob download may be blocked by Tauri CSP | +| 🟡 3 | Revise LOE to 108–112h range | Accounts for async branch, toast infra, JSON migration | + +--- + +*Full reviewer session:* `/Users/brandonkiefer/.omp/agent/sessions/-projects-atomic/2026-05-01T15-06-56-660Z_1accb47a-6b128d88-23d84057-7f51.jsonl` diff --git a/docs/plans/2026-05-01-health-dashboard-ui-improvements/plan.md b/docs/plans/2026-05-01-health-dashboard-ui-improvements/plan.md new file mode 100644 index 00000000..00a2107a --- /dev/null +++ b/docs/plans/2026-05-01-health-dashboard-ui-improvements/plan.md @@ -0,0 +1,972 @@ +# Knowledge Health Dashboard — UI Improvements + +**Date:** 2026-05-01 +**Status:** Reviewed (2026-05-01) — see [REVIEW.md](./REVIEW.md) +**Project:** Atomic (desktop + web) +**Request:** Implement comprehensive UX enhancements to the health dashboard component, including per-item actions, sample review, filtering, trending, and improved affordances. + +--- + +## Executive Summary + +The current health dashboard (`HealthWidget.tsx`) displays a vertical list of check rows with scores, summary text, and a global "Apply N fixes" button. This plan adds: + +1. **Per-row expandability** with Run and Review buttons, individual fix toggles +2. **Sample review panels** showing 3–5 atoms that triggered each issue, with quick-actions (Fix/Dismiss/Open) +3. **Score trending** (↑↓→ indicators) and last-run timestamps +4. **Filtering & sorting** (severity, auto-fixable, recency) +5. **Severity badges** (🔴🟠🟡🟢) and status colors per check +6. **Improved action bar** with confirmation modals, undo stack, export +7. **Micro-interactions** (animated bars, toast notifications, keyboard shortcuts) +8. **OpenAPI spec coverage** for all `/api/health/*` endpoints (prerequisite — currently missing) + +**Scope:** Frontend (React/TypeScript) + Backend (Rust: utoipa annotations + ApiDoc registration). Health endpoints are currently missing from the generated OpenAPI spec and must be added before external clients (iOS, Android, MCP, SDK consumers) can use them. + +**Effort:** 4 phases, ~100–110 hours of development (Phase 0 adds 8–10 hours for spec coverage). + +--- + +## Current Architecture & Evidence + +### HealthWidget.tsx (src/components/dashboard/widgets/) +- **Current structure:** + - Single component with hardcoded `CHECK_ORDER` (L160–L172) + - Per-row display: icon, label, score bar, description (L288–L332) + - Global "Apply N fixes" button with expandable checklist (L353–L383) + - Review modal dispatch (`HealthReviewModal`, L406–L410) + - No per-row timestamps, trending, or individual fix toggles + +- **Current state management:** + - `report`: Full `HealthReport` object with all checks + - `showPending`: Boolean toggle for "What will this do?" checklist + - `showReview`: Boolean for modal + - `lastFix`: Latest fix response result + - No per-check UI state (expanded, running, etc.) + +### HealthReviewModal.tsx +- Opens for one category at a time +- Shows pairs/samples and per-pair actions (Merge/Keep/Delete/Open) +- Uses `get_atom` API to fetch atom details +- No cross-category comparison or bulk operations + +### Backend API Surface (crates/atomic-server/src/routes/health.rs) +- `GET /api/health/knowledge` — Returns `HealthReport` with all checks + computed_at +- `POST /api/health/fix` — Takes `FixRequest { mode, include_medium, dry_run }` → returns `FixResponse` +- `POST /api/health/fix/{check}/{item_id}` — Manual per-item fix (merge/delete strategies) +- `POST /api/health/undo/{fix_id}` — Undo a fix from audit log +- `GET /api/health/history` — Recent stored reports (for trending) +- `GET /api/health/fixes/recent` — Recent fix log entries + +**Gap:** No endpoint for running a single check in isolation. Needed for per-row "Run" buttons. + +### Type Definitions (atomic-core/src/health/mod.rs) +- `HealthCheckResult`: `status`, `score`, `auto_fixable`, `requires_review`, `fix_action`, `data` +- `HealthReport`: `overall_score`, `overall_status`, `computed_at`, `checks: HashMap`, `auto_fixable`, `requires_review` +- `FixResponse`: `actions_taken: Vec`, `skipped`, `new_score` + +--- + +## Recommended Approach + +### Design Principles +1. **Preserve existing color scale** (green ≥90, yellow 70–89, orange 50–69, red <50) +2. **Dark theme (Obsidian-inspired):** `#1e1e1e` bg, `#7c3aed` purple accent +3. **Progressive disclosure:** Summary row → expandable for details → modal for complex decisions +4. **Idempotent actions:** All fixes are safely retryable +5. **Accessibility:** ARIA labels, focus states, keyboard navigation + +### Technical Strategy + +#### Phase 1: Foundation (Expandable rows, per-row state, Run/Review buttons) +- Refactor single `HealthWidget` into `HealthCheckRow` sub-component with local state +- Add `expandedChecks` Set to track which rows are open +- New endpoint: `POST /api/health/check/{check_name}` for isolated check runs +- Per-row buttons: Run (spinner), Review (lazy-load samples), individual fix toggle +- Local UI state: `lastRunTimes`, `checkTrends`, `checkSamples` + +#### Phase 2: Trends, Timestamps, Filtering (Score history, severity badges, sort/filter UI) +- Fetch historical reports from `GET /api/health/history` +- Compute score delta (current vs. previous) for trend indicator +- Add `last_run` timestamp to each check result (backend: store in report) +- Filter row above checks: Severity, Auto-fixable, Recency +- Sort options: By score (asc/desc), by affected count, alphabetical, auto-fixable first +- Severity badge logic: 🔴 (0–40), 🟠 (41–70), 🟡 (71–85), 🟢 (86–100) + +#### Phase 3: Advanced Affordances (Confirmation modals, undo stack, export, keyboard shortcuts) +- Confirmation modal before batch fixes (grouped by check, showing expected delta) +- Undo toast: "Undo" button + 10s timeout +- Export: Generate markdown report with all findings, citations, sample atoms +- Keyboard shortcuts: `r` (refresh), `1–7` (expand check), `f` (apply fixes), `?` (help) +- Animated score bars: CSS transition on mount/update +- Toast notifications: "✅ Fixed N items. Score 80 → 85" + +--- + +## Implementation Plan + +### Phase 0: OpenAPI Spec Coverage for Health Endpoints (Prerequisite, ~8–10 hours) + +**Why this is Phase 0:** The `export-openapi` binary and `utoipa` `ApiDoc` struct in `crates/atomic-server/src/lib.rs` drive spec generation for all external SDK consumers (iOS, Android, MCP bridge, third-party integrations). Every `/api/health/*` route is currently **missing from the spec** because: + +1. None of the handler functions in `crates/atomic-server/src/routes/health.rs` have `#[utoipa::path(...)]` attribute macros. +2. No health route paths are listed in the `#[openapi(paths(...))]` declaration in `crates/atomic-server/src/lib.rs` (lines 30–167). +3. Health-specific schema types (`HealthReport`, `HealthCheckResult`, `FixRequest`, `FixResponse`, `FixAction`, `SkippedFix`, `ManualFixRequest`, `HistoryQuery`, `StoredHealthReport`, `HealthFixLog`) are not in the `components(schemas(...))` list. + +**Evidence:** +- `crates/atomic-server/src/routes/health.rs` — handlers lack utoipa annotations (confirmed by reading the file; comments show routes but no `#[utoipa::path]` decorators) +- `crates/atomic-server/src/lib.rs:30–167` — paths list has no `routes::health::*` entries +- `crates/atomic-server/src/lib.rs:175–286` — components schemas list has no health types +- `crates/atomic-server/src/lib.rs:290–313` — tags list has no `health` entry + +**Impact of not fixing:** +- iOS/Android clients cannot generate typed bindings for health endpoints +- MCP tools for LLM agents cannot discover health operations +- External SDK consumers have no schema contract; they must reverse-engineer from handler code +- API reference docs (served at `/scalar`) omit the entire health surface + +#### 0.1 Annotate health route handlers with `#[utoipa::path(...)]` +**File:** `crates/atomic-server/src/routes/health.rs` + +Each handler needs a path macro. Example for `get_health_knowledge`: + +```rust +#[utoipa::path( + get, + path = "/api/health/knowledge", + tag = "health", + responses( + (status = 200, description = "Current health report", body = HealthReport), + (status = 500, description = "Internal server error", body = ApiErrorResponse), + ), + security(("bearer_auth" = [])), +)] +pub async fn get_health_knowledge(db: Db) -> HttpResponse { ... } +``` + +All seven handlers need annotation: + +| Handler | Method | Path | Notes | +|---------|--------|------|-------| +| `get_health_knowledge` | GET | `/api/health/knowledge` | Returns `HealthReport` | +| `run_health_fix` | POST | `/api/health/fix` | Body: `FixRequest`, returns `FixResponse` | +| `apply_manual_fix` | POST | `/api/health/fix/{check}/{item_id}` | Path params + `ManualFixRequest` body, returns `FixAction` or `{status: "no_op"}` | +| `undo_health_fix` | POST | `/api/health/undo/{fix_id}` | Path param, returns `{status, fix_id}` | +| `get_health_history` | GET | `/api/health/history` | Query: `limit`, returns `Vec` | +| `get_recent_fixes` | GET | `/api/health/fixes/recent` | Query: `limit`, returns `Vec` | +| `compute_single_check` (Phase 1 addition) | POST | `/api/health/check/{name}` | Path param, returns `(String, HealthCheckResult)` tuple | + +For path params and query params, add `params(...)` section to the macro. For the `{status: "no_op"}` literal-shape response, either define a typed `NoOpResponse` struct with `ToSchema` or use `body = Object` and document inline. + +#### 0.2 Add `ToSchema` derives to all health types +**File:** `crates/atomic-core/src/health/mod.rs` + +The struct definitions currently have `#[derive(Debug, Clone, Serialize, Deserialize)]`. Add `utoipa::ToSchema` using the feature-gated pattern already established in `atomic-core` (see `crates/atomic-core/src/models.rs`): + +```rust +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthCheckResult { ... } + +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthReport { ... } + +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FixAction { ... } + +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SkippedFix { ... } + +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FixResponse { ... } + +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FixRequest { ... } + +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum FixTier { Safe, Low, Medium, High } + +// Real variants — do NOT change them; only add the cfg_attr derive. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum HealthStatus { Healthy, NeedsAttention, Degraded, Unhealthy } +``` + +**File:** `crates/atomic-core/src/health/audit.rs` +```rust +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthFixLog { ... } + +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StoredHealthReport { ... } +``` + +`atomic-core` already has `utoipa` as an *optional* dep behind `[features] openapi = ["utoipa"]` in `crates/atomic-core/Cargo.toml`. **Do not add utoipa unconditionally** — use `cfg_attr` throughout so non-openapi consumers compile cleanly. `atomic-server` activates the feature via `atomic-core = { features = ["openapi", ...] }` in its own Cargo.toml. + +#### 0.3 Register health paths and schemas in ApiDoc +**File:** `crates/atomic-server/src/lib.rs` + +In the `#[openapi(paths(...))]` block (around line 30), add: + +```rust +// Health +routes::health::get_health_knowledge, +routes::health::run_health_fix, +routes::health::apply_manual_fix, +routes::health::undo_health_fix, +routes::health::get_health_history, +routes::health::get_recent_fixes, +routes::health::compute_single_check, // added in Phase 1 +``` + +In the `components(schemas(...))` block (around line 175), add: + +```rust +// Health +atomic_core::health::HealthReport, +atomic_core::health::HealthCheckResult, +atomic_core::health::HealthStatus, +atomic_core::health::FixRequest, +atomic_core::health::FixResponse, +atomic_core::health::FixAction, +atomic_core::health::SkippedFix, +atomic_core::health::FixTier, +atomic_core::health::audit::StoredHealthReport, +atomic_core::health::audit::HealthFixLog, +routes::health::ManualFixRequest, +``` + +In the `tags(...)` block (around line 290), add: + +```rust +(name = "health", description = "Knowledge base health checks and auto-remediation"), +``` + +#### 0.4 Verify spec generation + +Regenerate the OpenAPI JSON and confirm health endpoints appear: + +```bash +cargo run --bin export-openapi -p atomic-server -- openapi.json + +# Verify health paths are present +jq '.paths | keys | map(select(startswith("/api/health")))' openapi.json +# Expected: ["/api/health/check/{name}", "/api/health/fix", "/api/health/fix/{check}/{item_id}", +# "/api/health/fixes/recent", "/api/health/history", "/api/health/knowledge", +# "/api/health/undo/{fix_id}"] + +# Verify health schemas are registered +jq '.components.schemas | keys | map(select(startswith("Health") or startswith("Fix") or . == "ManualFixRequest"))' openapi.json +``` + +#### 0.5 Verify downstream consumers + +- Hit `/scalar` in dev mode — confirm health section renders with all 7 endpoints, each with request/response schemas +- Rebuild iOS/Android typed client bindings (if automated via codegen) and verify no compile errors +- MCP bridge: check that `atomic-mcp` discovers health tools if it reflects on the OpenAPI surface + +**Effort estimate:** 8–10 hours +- 2h — ToSchema derives on core types + Cargo.toml utoipa dep +- 3h — `#[utoipa::path]` annotations on all 6 existing handlers (plus the Phase 1 handler) +- 1h — `ApiDoc` registration in lib.rs +- 1h — spec regeneration, jq verification, `/scalar` smoke test +- 1–2h — fixing any `ToSchema` derivation issues (e.g., `HashMap` may need explicit schema hint; `DateTime` needs a format attribute) + +--- + +### Phase 1: Expandable Rows & Per-Check Actions (Week 1, ~35 hours) + +#### 1.1 Backend: New single-check compute endpoint +**File:** `crates/atomic-server/src/routes/health.rs` + +```rust +// POST /api/health/check/{check_name} +pub async fn compute_single_check( + db: Db, + path: web::Path, +) -> HttpResponse { + let check_name = path.into_inner(); + // Call atomic-core with just this check + match health::compute_single_check(&db.0, &check_name).await { + Ok(result) => HttpResponse::Ok().json(result), + Err(e) => crate::error::error_response(e), + } +} +``` + +**File:** `crates/atomic-core/src/health/mod.rs` + +```rust +/// Compute a single health check by name. +pub async fn compute_single_check( + core: &AtomicCore, + check_name: &str, +) -> Result<(String, HealthCheckResult), AtomicCoreError> { + let result = match check_name { + // Sync checks — fetch raw data once, dispatch to the appropriate fn + "embedding_coverage" + | "tagging_coverage" + | "content_overlap" + | "source_uniqueness" + | "wiki_coverage" + | "semantic_graph_freshness" + | "content_quality" + | "orphan_tags" + | "tag_health" + | "contradiction_detection" + | "boilerplate_pollution" => { + let raw = core.storage().health_check_data_sync().await?; + match check_name { + "embedding_coverage" => checks::embedding_coverage(&raw), + "tagging_coverage" => checks::tagging_coverage(&raw), + "content_overlap" => checks::content_overlap(&raw), + "source_uniqueness" => checks::source_uniqueness(&raw), + "wiki_coverage" => checks::wiki_coverage(&raw), + "semantic_graph_freshness" => checks::semantic_graph_freshness(&raw), + "content_quality" => checks::content_quality(&raw), + "orphan_tags" => checks::orphan_tags(&raw), + "tag_health" => checks::tag_health(&raw), + "contradiction_detection" => checks::contradiction_detection(&raw), + "boilerplate_pollution" => checks::boilerplate_pollution(&raw), + _ => unreachable!(), + } + } + // Async check — requires per-atom DB lookups + "broken_internal_links" => compute_link_check(core).await?, + _ => return Err(AtomicCoreError::Validation( + format!("Unknown health check: {}", check_name), + )), + }; + Ok((check_name.to_string(), result)) +} +``` + +**Backend routes registration:** Update `crates/atomic-server/src/routes/mod.rs` to add `POST /api/health/check/{check_name}` into the health scope (alongside the other health routes). + +#### 1.2 Frontend: Refactor to component-per-row +**File:** `src/components/dashboard/widgets/HealthCheckRow.tsx` (new) + +```typescript +interface HealthCheckRowProps { + checkName: string; + check: HealthCheckResult; + isExpanded: boolean; + onToggleExpand: (name: string) => void; + onRun: (name: string) => void; + onReview: (name: string) => void; + isRunning?: boolean; +} + +export function HealthCheckRow({ + checkName, + check, + isExpanded, + onToggleExpand, + onRun, + onReview, + isRunning, +}: HealthCheckRowProps) { + return ( +
+ {/* Header */} +
+ + + {/* Label & score */} +
+
+ + {CHECK_LABELS[checkName] ?? checkName} + + + {check.score} + +
+ +
+ + {/* Right-align buttons */} + + + {check.requires_review && ( + + )} + + +
+ + {/* Description */} + {!isExpanded && ( +

+ {CHECK_DESCRIPTIONS[checkName]?.(check.data)} +

+ )} + + {/* Expanded detail */} + {isExpanded && ( +
+

+ {CHECK_DESCRIPTIONS[checkName]?.(check.data)} +

+ + {check.auto_fixable && ( + + )} + + {check.requires_review && ( + + )} +
+ )} +
+ ); +} +``` + +**File:** `src/components/dashboard/widgets/HealthWidget.tsx` (refactored) + +```typescript +export function HealthPanel() { + const [report, setReport] = useState(null); + const [expandedChecks, setExpandedChecks] = useState>(new Set()); + const [runningCheck, setRunningCheck] = useState(null); + const [showReviewModal, setShowReviewModal] = useState(null); + // ... other state + + const runSingleCheck = useCallback(async (checkName: string) => { + setRunningCheck(checkName); + try { + const result = await getTransport().invoke<{ + name: string; + result: HealthCheckResult; + }>('health_check_single', { check_name: checkName }); + + // Update report with new check result + setReport((prev) => { + if (!prev) return prev; + return { + ...prev, + checks: { ...prev.checks, [checkName]: result.result }, + }; + }); + } catch (err) { + setError(err instanceof Error ? err.message : 'Check failed'); + } finally { + setRunningCheck(null); + } + }, []); + + const toggleExpandCheck = useCallback((checkName: string) => { + setExpandedChecks((prev) => { + const next = new Set(prev); + if (next.has(checkName)) next.delete(checkName); + else next.add(checkName); + return next; + }); + }, []); + + return ( +
+ {/* Header & score bar */} + {/* ... existing code ... */} + + {/* Per-check rows */} + {issueChecks.length > 0 ? ( +
+ {issueChecks.map((checkName) => { + const check = report.checks[checkName]; + if (!check) return null; + return ( + setShowReviewModal(name)} + isRunning={runningCheck === checkName} + /> + ); + })} +
+ ) : ( + /* healthy state */ + )} + + {showReviewModal && ( + setShowReviewModal(null)} + onResolved={fetchHealth} + /> + )} +
+ ); +} +``` + +**Command map:** Add `health_check_single` to `src/lib/transport/command-map.ts`. + +#### 1.3 Update HealthReviewModal for row-triggered opens +- Accept `checkName: string` as a new required prop alongside the existing `report`, `onClose`, and `onResolved` +- Use `checkName` to set initial `activeTab` state so the modal opens on the correct category +- Update the prop interface in `HealthReviewModal.tsx` to `{ report: HealthReport; checkName: string; onClose: () => void; onResolved: () => void }` + +**Effort estimate:** 35 hours (backend endpoint, TS types, component refactor, testing) + +--- + +### Phase 2: Trends, Filtering, Sorting (Week 2, ~30 hours) + +#### 2.1 Backend: Enhance HealthReport with metadata +**File:** `crates/atomic-core/src/health/mod.rs` + +```rust +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthReport { + pub overall_score: u32, + pub overall_status: String, // Keep as String — "healthy" | "needs_attention" | "degraded" | "unhealthy" + pub computed_at: String, // ISO 8601 + pub atom_count: i32, + pub checks: HashMap, + pub auto_fixable: i32, + pub requires_review: i32, + pub previous_score: Option, // Added in Phase 2 for trending; None on first run +} + +**File:** `crates/atomic-core/src/storage/sqlite/health.rs` + +Enhance `get_latest_health_report_impl` to fetch previous score from history table. + +#### 2.2 Frontend: Add trend computation and filter UI +**File:** `src/components/dashboard/widgets/HealthWidget.tsx` + +```typescript +interface FilterState { + severity: 'all' | 'critical' | 'warning' | 'needs-attention' | 'healthy'; + autoFixable: 'all' | 'fixable' | 'manual-only'; + sort: 'score-asc' | 'score-desc' | 'alphabetical' | 'affected-count'; +} + +// Trend indicator helper +function getTrend(check: HealthCheckResult, previousScore?: number): '↑' | '↓' | '→' { + if (!previousScore) return '→'; + if (check.score > previousScore) return '↑'; + if (check.score < previousScore) return '↓'; + return '→'; +} + +// Severity badge +function getSeverityBadge(score: number): string { + if (score <= 40) return '🔴'; + if (score <= 70) return '🟠'; + if (score <= 85) return '🟡'; + return '🟢'; +} + +// Filtered and sorted checks +function getVisibleChecks( + report: HealthReport, + filter: FilterState +): string[] { + let visible = CHECK_ORDER.filter((k) => { + const check = report.checks[k]; + if (!check || check.status === 'ok') return false; + + // Severity filter + if (filter.severity !== 'all') { + const score = check.score; + const severity = + score <= 40 ? 'critical' : + score <= 70 ? 'warning' : + score <= 85 ? 'needs-attention' : 'healthy'; + if (severity !== filter.severity) return false; + } + + // Auto-fixable filter + if (filter.autoFixable === 'fixable' && !check.auto_fixable) return false; + if (filter.autoFixable === 'manual-only' && check.auto_fixable) return false; + + return true; + }); + + // Sorting + if (filter.sort === 'score-asc') { + visible.sort((a, b) => report.checks[a].score - report.checks[b].score); + } else if (filter.sort === 'score-desc') { + visible.sort((a, b) => report.checks[b].score - report.checks[a].score); + } else if (filter.sort === 'alphabetical') { + visible.sort((a, b) => CHECK_LABELS[a].localeCompare(CHECK_LABELS[b])); + } else if (filter.sort === 'affected-count') { + visible.sort((a, b) => { + const countA = extractCount(report.checks[a]); + const countB = extractCount(report.checks[b]); + return countB - countA; + }); + } + + return visible; +} +``` + +#### 2.3 HealthCheckRow enhancement: Timestamps and trends +```typescript +// Update row header to show trend & last-run time +
+ {getTrend(check, previousScore)} + Last: 2h ago +
+ +// Severity badge before icon +{getSeverityBadge(check.score)} +``` + +**Effort estimate:** 30 hours (backend report enrichment, filter logic, sorting, UI layout) + +--- + +### Phase 3: Advanced UX (Modals, Undo, Export, Keyboard Shortcuts, Animations) (Week 3, ~25 hours) + +#### 3.1 Confirmation modal for batch fixes +```typescript +interface FixConfirmationModalProps { + pending: { label: string; check: string }[]; + report: HealthReport; + onConfirm: (selectedChecks: Set) => void; + onCancel: () => void; +} + +// Shows grouped summary: +// "This will: retag 26 atoms, remove 9 duplicate clones, trim 20 long atoms" +// With per-fix checkbox +``` + +#### 3.2 Undo stack & toast +```typescript +// Undo stack entries: each holds the fix_id (from HealthFixLog) and a human label +const [undoStack, setUndoStack] = useState<{ fix_id: string; label: string }[]>([]); + +// After fix applied, push { fix_id, label } onto the stack and show toast +// fix_id comes from the HealthFixLog.id returned by log_fix (server includes it in FixResponse.fix_id) +// Toast auto-dismisses in 10s (clearTimeout on click); Undo calls POST /api/health/undo/{fix_id} + +#### 3.3 Export to markdown +```typescript +function exportHealthReport(report: HealthReport): string { + let md = `# Knowledge Base Health Report\n\n`; + md += `**Overall Score:** ${report.overall_score}/100\n`; + md += `**Generated:** ${new Date(report.computed_at).toLocaleString()}\n\n`; + + // Per-check section with data + for (const check of CHECK_ORDER) { + const result = report.checks[check]; + if (!result) continue; + md += `## ${CHECK_LABELS[check]}\n`; + md += `**Score:** ${result.score}/100\n`; + md += `**Status:** ${result.status}\n`; + md += `${CHECK_DESCRIPTIONS[check]?.(result.data)}\n\n`; + } + + return md; +} +``` + +#### 3.4 Keyboard shortcuts +- `r`: Refresh all checks +- `f`: Apply fixes (open confirmation modal) +- `1–9`: Expand nth check in filtered list +- `?`: Show help overlay + +#### 3.5 Animations +```css +/* Smooth score bar fill */ +.score-bar { + transition: width 600ms cubic-bezier(0.34, 1.56, 0.64, 1); /* ease-out */ +} + +/* Row expand/collapse */ +[data-expanded="true"] { + animation: slideDown 200ms ease-out; +} + +@keyframes slideDown { + from { + opacity: 0; + transform: translateY(-8px); + } + to { + opacity: 1; + transform: translateY(0); + } +} +``` + +**Effort estimate:** 25 hours (modals, UX polish, keyboard handling, animations) + +--- + +## Files & Components to Change + +### Backend +- `crates/atomic-server/src/routes/health.rs` — Add `compute_single_check` endpoint; annotate all handlers with `#[utoipa::path(...)]` (Phase 0) +- `crates/atomic-server/src/lib.rs` — Register health paths, schemas, and tag in `ApiDoc` (Phase 0) +- `crates/atomic-core/src/health/mod.rs` — `compute_single_check()` function + `ToSchema` derives on all health types (Phase 0) +- `crates/atomic-core/src/health/audit.rs` — `ToSchema` derives on `HealthFixLog`, `StoredHealthReport` (Phase 0) +- `crates/atomic-core/Cargo.toml` — Add `utoipa` dependency (Phase 0) +- `crates/atomic-core/src/storage/sqlite/health.rs` — Query previous report score for trending +- `crates/atomic-server/src/routes/mod.rs` — Route registration + +### Frontend +- `src/components/dashboard/widgets/HealthWidget.tsx` — Main refactor (expandable, filtering, actions) +- `src/components/dashboard/widgets/HealthCheckRow.tsx` — NEW (per-row component) +- `src/components/dashboard/widgets/HealthReviewModal.tsx` — Minor: accept checkName prop +- `src/components/dashboard/widgets/HealthConfirmModal.tsx` — NEW (batch fix confirmation) +- `src/components/dashboard/widgets/HealthExportModal.tsx` — NEW (markdown export) +- `src/lib/transport/command-map.ts` — Add `health_check_single` command +- `src/styles/animations.css` — NEW or extend (score bar animations) + +--- + +## Data Flow & Interfaces + +### Single-Check Compute Flow +``` +User clicks "Run" button on Tagging row + → onRun('tagging_coverage') + → POST /api/health/check/tagging_coverage + → compute_single_check(core, 'tagging_coverage') + → fetch raw data, run just tagging check + → return HealthCheckResult + → update report.checks['tagging_coverage'] + → row re-renders with new score, animate bar +``` + +### Batch Fix with Confirmation +``` +User clicks "Apply N fixes" + → open FixConfirmationModal + → show checklist of pending.map(fix_action) + → user can toggle individual fixes + → user clicks "Confirm" + → POST /api/health/fix { mode: 'auto', include_medium, dry_run } + → FixResponse + new_score + → update report + → show toast: "✅ Fixed 5 items. Score 80 → 85" + → Undo button available for 10s +``` + +### Trend Computation +``` +GET /api/health/knowledge + → HealthReport { overall_score, checks, computed_at, previous_score? } + → for each check, if previous_score exists: + delta = current - previous + trend = delta > 0 ? '↑' : delta < 0 ? '↓' : '→' + → display trend icon next to score +``` + +--- + +## Configuration & Deployment Notes + +### Environment +- No new env vars needed; all features toggle on frontend state +- Backend endpoint (`compute_single_check`) available on all deployments + +### Feature Flags +None required; all features are additive and don't conflict with existing UI. + +### Accessibility +- All buttons have aria-labels and keyboard focus states +- Modals use `dialog` ARIA role +- Color not the only indicator (use icons + text) +- Keyboard shortcuts documented in `?` overlay + +--- + +## Testing & Validation Plan + +### E2E Tests (Playwright) +1. **Per-row run button** + - Click Run on a single check + - Verify spinner appears + - Verify score updates when response arrives + - Verify can click Run multiple times without errors + +2. **Expandable rows** + - Click row header → expands and shows details + buttons + - Click again → collapses + - Expand state persists until user collapses + +3. **Batch fix confirmation** + - Click "Apply N fixes" → modal opens + - Each fix has unchecked checkbox + - User can toggle individual fixes + - Click Confirm → fixes run, new score displayed + - Toast shows "Fixed N items. Score X → Y" + - Click Undo → reverts (calls undo endpoint) + +4. **Filtering & sorting** + - Change severity filter → only matching checks displayed + - Change sort order → checks reorder + - Verify all checks still displayed when filter cleared + +5. **Sample review** + - Click "Review samples" on a failing check + - Modal opens showing 3–5 sample atoms + - Each sample has "Fix", "Dismiss", "Open atom" buttons + - Quick actions work as expected + +### Commands +```bash +# Run E2E tests +npm run playwright:test -- --grep "health.*ui" + +# Run unit tests for helper functions +npm run test -- HealthWidget + +# Manual testing flow +npm run dev:mobile:ios & +# or +make dev-desktop-fast + +# In app: +1. Navigate to dashboard +2. Open Health panel +3. Test each UI interaction as per E2E list above +``` + +### Verification +- All checks remain sortable/filterable after batch fix +- Score bars animate smoothly on update +- Keyboard shortcuts work (test with `r`, `f`, `?`) +- No console errors during interactions +- Modal accessibility tested with screen reader (NVDA/Voiceover) + +--- + +## Risks, Assumptions, and Open Questions + +### Risks +1. **Performance:** Fetching `HealthReport` + historical data on every refresh could be slow for large KBs. Mitigation: Memoize last report, fetch history lazily on first filter/trend request. + +2. **Undo semantics:** If user applies fixes, then runs a check that changes scores, what happens to undo? Fix: Undo button is only valid for 10s immediately after fix; once new data fetched, undo is stale. + +3. **Concurrent fixes:** User clicks "Run" on a single check while batch fix is running. Mitigation: Disable Run buttons while batch fix in progress. + +### Assumptions +1. Backend will have `POST /api/health/check/{check_name}` endpoint for single-check compute (requirement for Phase 1). +2. History API already exists and returns previous reports (assumed; verify with backend team). +3. Undo endpoint (`POST /api/health/undo/{fix_id}`) already works (implemented in Phase 1 of prior sprint). +4. HealthCheckResult data shape is stable; no breaking changes to check payloads. + +### Open Questions +1. **Sample review:** Currently `HealthReviewModal` shows pairs for overlaps/dupes. For other checks (e.g., untagged atoms, too-long atoms), how should samples be structured? Should we add a new `/api/health/samples/{check}` endpoint? + +2. **Export location:** Where should markdown export be saved? Recommendation: web uses `` with a `data:text/markdown` URI; Tauri desktop uses `@tauri-apps/plugin-dialog` (`save()`) + `@tauri-apps/plugin-fs` (`writeTextFile()`) to avoid CSP-blocked blob downloads. Branch on `window.__TAURI__` at runtime. + +3. **Trend baseline:** Should we compare to the *previous run* or a *rolling average* over 7 days? Recommend previous run (simpler, clearer signal). + +4. **Mobile:** Filter/sort UI is complex on mobile. Should we hide advanced filters on small screens and show only "Severity" dropdown? Or move to slide-over panel? + +--- + +## LOE & Effort Estimate + +| Phase | Task | Hours | Notes | +|-------|------|-------|-------| +| 0 | Backend: `cfg_attr` ToSchema derives on health core types | 2 | Feature-gated pattern; ~10 structs/enums in mod.rs + audit.rs | +| 0 | Backend: `#[utoipa::path]` on health handlers | 3 | Six existing + one Phase 1 addition | +| 0 | Backend: Register paths/schemas/tags in ApiDoc | 1 | Edit `atomic-server/src/lib.rs` | +| 0 | Verify generated spec, regen clients | 2 | `export-openapi`, jq checks, `/scalar` smoke test | +| 0 | Fix ToSchema derivation edge cases | 2 | DateTime formats, HashMap schema hints | +| **Phase 0 total** | | **10** | Prerequisite — must ship before external clients use health APIs | +| 1 | Backend: single-check endpoint (all 11 + async link check) | 10 | Full dispatch matrix + `broken_internal_links` async path | +| 1 | Frontend: HealthCheckRow component | 12 | Component extraction, state per-row, buttons | +| 1 | Frontend: HealthWidget refactor | 10 | Migrate to per-row render, integrate new state | +| 1 | Integration testing, fixes | 6 | E2E tests for Run, Review, expand/collapse | +| **Phase 1 total** | | **38** | | +| 2 | Backend: report enrichment (prev score) | 7 | Query history, add field, handle first-run NULL, update TS interface | +| 2 | Frontend: filter/sort logic | 10 | Data structures, comparison functions | +| 2 | Frontend: filter UI, severity badges | 8 | Layout, state management | +| 2 | Integration testing | 7 | Filter combinations, sorting verified | +| **Phase 2 total** | | **32** | | +| 3 | FixConfirmationModal component | 6 | Modal boilerplate, checkbox logic | +| 3 | Undo stack & toast integration | 7 | Toast library setup, undo flow, aria-live, timeout cleanup | +| 3 | Export function + UI (web + Tauri paths) | 5 | Markdown generation, plugin-fs for desktop, data: for web | +| 3 | Keyboard shortcuts | 3 | Event listeners, help overlay | +| 3 | Animations & micro-interactions | 4 | CSS transitions, polish | +| 3 | E2E and polish | 3 | Final testing, edge cases | +| **Phase 3 total** | | **28** | | +| **Grand total** | | **108** | ~3–4 weeks at 30 hrs/week (Phase 0 can run in parallel with Phase 1 frontend work once `cfg_attr` pattern settled) | + +--- + +## Decision Log + +| Date | Decision | Rationale | +|------|----------|-----------| +| 2026-05-01 | Phased delivery (foundation → trends → polish) | Allows early feedback and MVP deployment after Phase 1 | +| 2026-05-01 | Per-row Run button over modal | Faster iteration; avoid extra click depth for common operation | +| 2026-05-01 | Severity filter over custom query language | Simpler UX; covers 90% of user needs | +| 2026-05-01 | 10s undo timeout vs. infinite stack | Prevents confusion; aligns with user mental model (like Ctrl+Z in editors) | +| 2026-05-01 | Markdown export vs. JSON/CSV | Markdown is human-readable, shareable, LLM-friendly for context | +| 2026-05-01 | Add Phase 0 for OpenAPI spec coverage | Health endpoints are invisible to external SDK/MCP/iOS clients until registered in `ApiDoc`; Phase 0 unblocks all downstream consumers and is a prerequisite for Phase 1's new `compute_single_check` endpoint being usable outside the web UI | +| 2026-05-01 | Use `cfg_attr(feature = "openapi", derive(utoipa::ToSchema))` for health types | Matches existing atomic-core convention (models.rs); utoipa is already an optional dep behind `openapi` feature — unconditional derive breaks non-openapi consumers | + +--- + +## Summary + +This plan structures a high-UX dashboard enhancement into four phased releases: + +0. **Phase 0 (Prerequisite):** OpenAPI spec coverage for all `/api/health/*` endpoints — adds utoipa annotations, `ToSchema` derives, and `ApiDoc` registration so external SDK/MCP/mobile clients can consume health APIs. +1. **Phase 1 (MVP):** Expandable rows with Run/Review per-check, lays foundation for remaining features. +2. **Phase 2 (Insight):** Trending, filtering, sorting so users can prioritize high-impact fixes. +3. **Phase 3 (Polish):** Confirmations, undo, export, keyboard shortcuts, animations — delightful UX. + +**Recommendation:** Start Phase 0 immediately in parallel with Phase 1 frontend work — backend annotation work doesn't block React refactor. Settle the `cfg_attr` pattern (Phase 0 §0.2) before merging Phase 1 to avoid feature-flag conflicts. Phase 2 and 3 follow incrementally. + +Estimated **~108 hours total** across all four phases, with Phase 0 (~10h) deliverable within a day or two and unblocking all external API consumers. diff --git a/src/components/dashboard/registry.ts b/src/components/dashboard/registry.ts index aee78266..53d78063 100644 --- a/src/components/dashboard/registry.ts +++ b/src/components/dashboard/registry.ts @@ -2,7 +2,9 @@ import type { FC } from 'react'; import { BriefingWidget } from './widgets/BriefingWidget'; import { ActivityWidget } from './widgets/ActivityWidget'; import { NewWikisWidget } from './widgets/NewWikisWidget'; +import { RecentWikisWidget } from './widgets/RecentWikisWidget'; import { RevisionsWidget } from './widgets/RevisionsWidget'; +import { HealthPanel } from './widgets/HealthWidget'; export type WidgetSpan = 'full' | 'half'; @@ -13,8 +15,10 @@ export interface DashboardWidget { } export const dashboardWidgets: DashboardWidget[] = [ - { id: 'briefing', span: 'full', Component: BriefingWidget }, - { id: 'activity', span: 'half', Component: ActivityWidget }, - { id: 'new-wikis', span: 'half', Component: NewWikisWidget }, - { id: 'revisions', span: 'full', Component: RevisionsWidget }, -]; + { id: 'briefing', span: 'full', Component: BriefingWidget }, + { id: 'activity', span: 'half', Component: ActivityWidget }, + { id: 'new-wikis', span: 'half', Component: NewWikisWidget }, + { id: 'recent-wikis', span: 'half', Component: RecentWikisWidget }, + { id: 'revisions', span: 'half', Component: RevisionsWidget }, + { id: 'health', span: 'full', Component: HealthPanel }, +]; \ No newline at end of file diff --git a/src/components/dashboard/widgets/HealthCheckRow.tsx b/src/components/dashboard/widgets/HealthCheckRow.tsx new file mode 100644 index 00000000..96a4ee61 --- /dev/null +++ b/src/components/dashboard/widgets/HealthCheckRow.tsx @@ -0,0 +1,169 @@ +import { Play, Search, ChevronDown, ChevronUp, Loader2 } from 'lucide-react'; + +export function getTrend(current: number, previous?: number): '↑' | '↓' | '→' { + if (previous === undefined) return '→'; + if (current > previous) return '↑'; + if (current < previous) return '↓'; + return '→'; +} +// These constants are re-exported here for use by HealthCheckRow +// The canonical source remains HealthWidget.tsx +export interface HealthCheckResult { + status: 'ok' | 'warning' | 'error' | 'info'; + score: number; + auto_fixable: boolean; + requires_review: boolean; + fix_action?: string; + data: Record; +} + +export interface HealthCheckRowProps { + checkName: string; + check: HealthCheckResult; + label: string; + description: string; + isExpanded: boolean; + onToggleExpand: (name: string) => void; + onRun: (name: string) => void; + onReview: (name: string) => void; + isRunning: boolean; + includeInFix: boolean; + onToggleInclude: (name: string) => void; + trend?: '↑' | '↓' | '→'; + severityBadge?: string; +} + +function ScoreBarMini({ score }: { score: number }) { + const color = + score >= 90 ? 'bg-green-500' : + score >= 70 ? 'bg-yellow-500' : + score >= 50 ? 'bg-orange-500' : 'bg-red-500'; + return ( +
+
+
+ ); +} + +export function HealthCheckRow({ + checkName, + check, + label, + description, + isExpanded, + onToggleExpand, + onRun, + onReview, + isRunning, + includeInFix, + onToggleInclude, + trend, + severityBadge, +}: HealthCheckRowProps) { + const scoreColor = + check.score >= 90 ? 'text-green-400' : + check.score >= 70 ? 'text-yellow-400' : + check.score >= 50 ? 'text-orange-400' : 'text-red-400'; + + return ( +
+ {/* Header row */} +
+ {/* Expand toggle */} + + + {/* Label + score bar */} +
+ {label} + + {check.score} +
+ + {/* Trend indicator */} + {trend !== undefined && ( + + {trend} + + )} + {severityBadge && ( + {severityBadge} + )} + {/* Action buttons */} +
+ + + {check.requires_review && ( + + )} +
+
+ + {/* Description (always shown) */} + {description && ( +

{description}

+ )} + + {/* Expanded detail */} + {isExpanded && ( +
+ {check.auto_fixable && ( + + )} + + {check.requires_review && ( + + )} +
+ )} +
+ ); +} diff --git a/src/components/dashboard/widgets/HealthReviewModal.tsx b/src/components/dashboard/widgets/HealthReviewModal.tsx new file mode 100644 index 00000000..d112cd9e --- /dev/null +++ b/src/components/dashboard/widgets/HealthReviewModal.tsx @@ -0,0 +1,455 @@ +import { useState, useEffect, useCallback } from 'react'; +import { createPortal } from 'react-dom'; +import { + X, GitMerge, Link, Trash2, Loader2, CheckCircle, + ChevronDown, ChevronUp, ExternalLink, RefreshCw, +} from 'lucide-react'; +import { getTransport } from '../../../lib/transport'; + +// ==================== Types ==================== + +export interface OverlapPair { + pair_id: string; + atom_a: { id: string; title: string; source?: string }; + atom_b: { id: string; title: string; source?: string }; + similarity: number; + shared_tag_count: number; + available_actions: string[]; +} + +interface AtomDetail { + id: string; + content: string; + source_url?: string; +} + +type PairAction = 'merge_with_llm' | 'keep_both' | 'delete_older'; +type PairStatus = 'idle' | 'loading' | 'done' | 'error'; + +// ==================== Helpers ==================== + +function sourceLabel(source?: string): string { + if (!source) return 'manual'; + try { return new URL(source).hostname; } catch { return source.split('/').slice(0, 2).join('/'); } +} + +function similarityLabel(s: number): { text: string; color: string } { + if (s >= 0.80) return { text: `${(s * 100).toFixed(0)}% overlap`, color: 'text-orange-400' }; + if (s >= 0.65) return { text: `${(s * 100).toFixed(0)}% overlap`, color: 'text-yellow-400' }; + return { text: `${(s * 100).toFixed(0)}% overlap`, color: 'text-gray-400' }; +} + +// ==================== Overlap pair row ==================== + +function PairRow({ + pair, + onApply, +}: { + pair: OverlapPair; + onApply: (pair: OverlapPair, action: PairAction) => Promise; +}) { + const [status, setStatus] = useState('idle'); + const [appliedAction, setAppliedAction] = useState(null); + const [error, setError] = useState(null); + const [expanded, setExpanded] = useState(false); + const [contents, setContents] = useState<[string, string] | null>(null); + const [loadingContent, setLoadingContent] = useState(false); + const sim = similarityLabel(pair.similarity); + + const apply = async (action: PairAction) => { + setStatus('loading'); + setAppliedAction(action); + setError(null); + try { + await onApply(pair, action); + setStatus('done'); + } catch (e) { + setStatus('error'); + setError(e instanceof Error ? e.message : 'Action failed'); + } + }; + + const toggleExpand = async () => { + if (!expanded && !contents) { + setLoadingContent(true); + try { + const [a, b] = await Promise.all([ + getTransport().invoke('get_atom', { id: pair.atom_a.id }), + getTransport().invoke('get_atom', { id: pair.atom_b.id }), + ]); + setContents([a.content, b.content]); + } finally { + setLoadingContent(false); + } + } + setExpanded(v => !v); + }; + + if (status === 'done') { + const labels: Record = { + merge_with_llm: 'Merged — LLM synthesised both atoms into one', + keep_both: 'Kept both — no changes made', + delete_older: 'Older atom deleted', + }; + return ( +
+ + {labels[appliedAction!]} +
+ ); + } + + return ( +
+
+ {/* Header row */} +
+ {sim.text} +
+ {pair.shared_tag_count > 0 && ( + {pair.shared_tag_count} shared tag{pair.shared_tag_count !== 1 ? 's' : ''} + )} + +
+
+ + {/* Atom summaries */} +
+ {[pair.atom_a, pair.atom_b].map((atom, i) => ( +
+

{atom.title}

+

{sourceLabel(atom.source)}

+
+ ))} +
+ + {/* Side-by-side content */} + {expanded && contents && ( +
+ {[pair.atom_a, pair.atom_b].map((atom, i) => ( +
+

{atom.title}

+ 
+                  {contents[i as 0 | 1]}
+
+
+ ))} +
+ )} + + {/* Error */} + {error &&

{error}

} + + {/* Actions */} +
+ } + label="Merge" + title="LLM synthesises both into one atom, preserving all unique content" + loading={status === 'loading' && appliedAction === 'merge_with_llm'} + disabled={status === 'loading'} + onClick={() => apply('merge_with_llm')} + /> + } + label="Keep both" + title="Leave both atoms — different perspectives on the same topic" + loading={status === 'loading' && appliedAction === 'keep_both'} + disabled={status === 'loading'} + onClick={() => apply('keep_both')} + /> + } + label="Delete older" + title="Delete the older atom" + loading={status === 'loading' && appliedAction === 'delete_older'} + disabled={status === 'loading'} + variant="danger" + onClick={() => apply('delete_older')} + /> +
+
+
+ ); +} + +function ActionBtn({ + icon, label, title, loading, disabled, onClick, variant = 'default', +}: { + icon: React.ReactNode; label: string; title: string; + loading: boolean; disabled: boolean; onClick: () => void; + variant?: 'default' | 'danger'; +}) { + return ( + + ); +} + +// ==================== Boilerplate section ==================== + +interface BoilerplateAtom { + id: string; + title: string; + source_url: string | null; + reembedStatus: 'idle' | 'loading' | 'done' | 'error'; +} + +function BoilerplateSection({ atomIds }: { atomIds: string[] }) { + const [atoms, setAtoms] = useState([]); + const [loadingAtoms, setLoadingAtoms] = useState(true); + + useEffect(() => { + let cancelled = false; + const fetchAll = async () => { + setLoadingAtoms(true); + const results = await Promise.allSettled( + atomIds.map(id => getTransport().invoke<{ id: string; content: string; source_url?: string }>('get_atom', { id })) + ); + if (cancelled) return; + setAtoms(results.map((r, i) => { + if (r.status === 'fulfilled') { + const first_line = r.value.content.split('\n').find(l => l.trim()) ?? atomIds[i]; + const title = first_line.replace(/^#+\s*/, '').trim().slice(0, 80); + return { id: atomIds[i], title, source_url: r.value.source_url ?? null, reembedStatus: 'idle' }; + } + return { id: atomIds[i], title: atomIds[i], source_url: null, reembedStatus: 'idle' }; + })); + setLoadingAtoms(false); + }; + fetchAll(); + return () => { cancelled = true; }; + }, [atomIds]); + + const reembed = async (atomId: string) => { + setAtoms(prev => prev.map(a => a.id === atomId ? { ...a, reembedStatus: 'loading' } : a)); + try { + await getTransport().invoke('retry_embedding', { atomId: atomId }); + setAtoms(prev => prev.map(a => a.id === atomId ? { ...a, reembedStatus: 'done' } : a)); + } catch { + setAtoms(prev => prev.map(a => a.id === atomId ? { ...a, reembedStatus: 'error' } : a)); + } + }; + + if (loadingAtoms) { + return ( +
+ +
+ ); + } + + return ( +
+
+

Embedding quality issue

+

+ These {atomIds.length} atoms share identical boilerplate sections that dominate their + embeddings — semantic search cannot reliably distinguish them from each other. + Edit each atom to remove or uniquify the boilerplate sections, then re-embed. +

+
+ +
+ {atoms.map(atom => ( +
+
+

{atom.title}

+ {atom.source_url && ( +

{sourceLabel(atom.source_url)}

+ )} +
+
+ {atom.source_url && ( + + + Source + + )} + {atom.reembedStatus === 'done' ? ( + + Queued + + ) : ( + + )} +
+
+ ))} +
+
+ ); +} + +// ==================== Modal ==================== + +interface Props { + report: { + checks: Record; + }>; + }; + checkName?: string; // If provided, pre-select this tab on open + onClose: () => void; + onResolved: () => void; +} + +export function HealthReviewModal({ report, checkName, onClose, onResolved }: Props) { + // Compute once — stable references for the lifetime of this modal mount + const overlapPairs: OverlapPair[] = + (report.checks['content_overlap']?.data?.pairs as OverlapPair[]) ?? []; + const boilerplateIds: string[] = + (report.checks['boilerplate_pollution']?.data?.affected_atoms as string[]) ?? []; + + // Build tab list from available data + const tabs = [ + ...(overlapPairs.length > 0 ? [{ key: 'content_overlap', label: 'Content overlap', count: overlapPairs.length }] : []), + ...(boilerplateIds.length > 0 ? [{ key: 'boilerplate', label: 'Boilerplate', count: boilerplateIds.length }] : []), + ]; + + // selectedTab = user choice; falls back to first available tab + const [selectedTab, setSelectedTab] = useState(checkName ?? null); + const activeTab = tabs.find(t => t.key === selectedTab)?.key ?? tabs[0]?.key ?? null; + + const [resolvedCount, setResolvedCount] = useState(0); + + useEffect(() => { + const handler = (e: KeyboardEvent) => { if (e.key === 'Escape') onClose(); }; + document.addEventListener('keydown', handler); + document.body.style.overflow = 'hidden'; + return () => { + document.removeEventListener('keydown', handler); + document.body.style.overflow = ''; + }; + }, [onClose]); + + const applyPairFix = useCallback(async (pair: OverlapPair, action: PairAction) => { + if (action === 'keep_both') { + setResolvedCount(n => n + 1); + return; + } + const itemId = `${pair.atom_a.id}_${pair.atom_b.id}`; + await getTransport().invoke('apply_health_item_fix', { + check: 'duplicate_detection', + item_id: itemId, + action, + }); + setResolvedCount(n => n + 1); + onResolved(); + }, [onResolved]); + + return createPortal( +
{ if (e.target === e.currentTarget) onClose(); }} + > +
+ + {/* Header */} +
+
+

Review Queue

+

+ {resolvedCount > 0 + ? `${resolvedCount} resolved this session` + : 'Items that need a judgment call'} +

+
+ +
+ + {/* Tabs */} + {tabs.length > 1 && ( +
+ {tabs.map(t => ( + + ))} +
+ )} + + {/* Content */} +
+ + {activeTab === null && ( +

Nothing to review — all clear

+ )} + + {activeTab === 'content_overlap' && ( + <> +

+ Atoms from different sources with 55–85% similarity and at least 2 shared tags. + These likely cover the same topic from different angles. + Use Keep both for complementary perspectives,{' '} + Merge for true duplicates. +

+ {overlapPairs.map(pair => ( + + ))} + + )} + + {activeTab === 'boilerplate' && ( + + )} + +
+
+
, + document.body, + ); +} diff --git a/src/components/dashboard/widgets/HealthWidget.tsx b/src/components/dashboard/widgets/HealthWidget.tsx new file mode 100644 index 00000000..9e2173e6 --- /dev/null +++ b/src/components/dashboard/widgets/HealthWidget.tsx @@ -0,0 +1,597 @@ +import { useEffect, useState, useCallback } from 'react'; +import { getTransport } from '../../../lib/transport'; +import { + RefreshCw, CheckCircle, AlertTriangle, XCircle, Play, +} from 'lucide-react'; +import { HealthReviewModal } from './HealthReviewModal'; +import { HealthCheckRow, getTrend } from './HealthCheckRow'; +import type { HealthCheckResult } from './HealthCheckRow'; + +// ==================== Types ==================== + +interface HealthReport { + overall_score: number; + overall_status: 'healthy' | 'needs_attention' | 'degraded' | 'unhealthy'; + computed_at: string; + atom_count: number; + checks: Record; + auto_fixable: number; + requires_review: number; + previous_score?: number; + previous_check_scores?: Record; +} + +interface FixAction { + id: string; + check: string; + action: string; + count: number; + details: string[]; +} + +interface FixResponse { + mode: string; + actions_taken: FixAction[]; + skipped: Array<{ check: string; reason: string; count: number }>; + new_score: number; +} + +// ==================== Config ==================== + +const CHECK_LABELS: Record = { + embedding_coverage: 'Embeddings', + tagging_coverage: 'Tagging', + source_uniqueness: 'Source dupes', + orphan_tags: 'Orphan tags', + semantic_graph_freshness: 'Semantic graph', + wiki_coverage: 'Wiki coverage', + content_quality: 'Content quality', + tag_health: 'Tag health', + duplicate_detection: 'Duplicates', + content_overlap: 'Content overlap', + contradiction_detection: 'Contradictions', + boilerplate_pollution: 'Boilerplate', + broken_internal_links: 'Broken links', +}; + +// One-line explanation shown under each failing check +const CHECK_DESCRIPTIONS: Record) => string> = { + embedding_coverage: (d) => { + const failed = d.failed as number ?? 0; + const pending = d.pending as number ?? 0; + if (failed > 0) return `${failed} atom${failed !== 1 ? 's' : ''} failed to embed — semantic search can't find them`; + if (pending > 0) return `${pending} atom${pending !== 1 ? 's' : ''} waiting to be embedded`; + return 'All atoms are embedded'; + }, + tagging_coverage: (d) => { + const untagged = (d.untagged_complete as number ?? 0) + (d.skipped_untagged as number ?? 0); + const failed = d.failed as number ?? 0; + if (untagged > 0) return `${untagged} atom${untagged !== 1 ? 's' : ''} went through the tagger but got zero tags assigned`; + if (failed > 0) return `${failed} atom${failed !== 1 ? 's' : ''} failed tagging`; + return 'All atoms are tagged'; + }, + source_uniqueness: (d) => { + const count = d.count as number ?? 0; + return `${count} source URL${count !== 1 ? 's' : ''} appear on more than one atom — likely an import bug`; + }, + orphan_tags: (d) => { + const count = (d.tags as unknown[])?.length ?? d.count as number ?? 0; + return `${count} tag${count !== 1 ? 's' : ''} with no atoms and no children — clutter in the tag tree`; + }, + semantic_graph_freshness: (d) => { + const n = d.atoms_since_rebuild as number ?? 0; + return `${n} atom${n !== 1 ? 's' : ''} added or updated since the similarity graph was last built`; + }, + wiki_coverage: (d) => { + const missing = d.without_wiki as number ?? 0; + const stale = d.stale_wikis as number ?? 0; + const parts = []; + if (missing > 0) parts.push(`${missing} eligible tag${missing !== 1 ? 's' : ''} have no wiki`); + if (stale > 0) parts.push(`${stale} wiki${stale !== 1 ? 's' : ''} are out of date`); + return parts.join(', '); + }, + content_quality: (d) => { + const issues = d.issues as Record | undefined; + if (!issues) return 'Some atoms may need attention'; + const parts = []; + if (issues.very_short?.count > 0) parts.push(`${issues.very_short.count} too short`); + if (issues.very_long?.count > 0) parts.push(`${issues.very_long.count} too long`); + if (issues.no_headings?.count > 0) parts.push(`${issues.no_headings.count} lack headings`); + if (issues.no_source?.count > 0) parts.push(`${issues.no_source.count} have no source`); + return parts.join(', '); + }, + tag_health: (d) => { + const parts = []; + if ((d.single_atom_tags as number) > 3) parts.push(`${d.single_atom_tags} single-atom tags`); + if ((d.rootless_tags as number) > 0) parts.push(`${d.rootless_tags} root-level tags may need nesting`); + if ((d.similar_name_pairs as number) > 0) parts.push(`${d.similar_name_pairs} similar-name pairs`); + return parts.join(', ') || 'Tag structure has issues'; + }, + content_overlap: (d) => { + const overlaps = (d.cross_source_overlaps as number) ?? 0; + const exact = (d.exact_duplicates as number) ?? 0; + const templates = (d.template_clones as number) ?? 0; + const parts = []; + if (exact > 0) parts.push(`${exact} exact URL duplicate${exact !== 1 ? 's' : ''}`); + if (templates > 0) parts.push(`${templates} template clone${templates !== 1 ? 's' : ''}`); + if (overlaps > 0) parts.push(`${overlaps} cross-source overlap${overlaps !== 1 ? 's' : ''} need review`); + return parts.join(', ') || 'No cross-source content overlap'; + }, + contradiction_detection: (d) => { + const count = d.potential_contradictions as number ?? 0; + return `${count} atom pair${count !== 1 ? 's' : ''} on the same topic with differing content`; + }, + boilerplate_pollution: (d) => { + const count = d.count as number ?? 0; + return `${count} atom${count !== 1 ? 's' : ''} share so much template text that semantic search can't distinguish them`; + }, + broken_internal_links: (d) => { + const n = (d.broken_count as number) ?? 0; + const atoms = (d.affected_atoms as number) ?? 0; + return `${n} link${n !== 1 ? 's' : ''} in ${atoms} atom${atoms !== 1 ? 's' : ''} point to other vault documents but resolve to no atom`; + }, +}; + +// Human-readable description of each fix_action value +const FIX_ACTION_LABELS: Record = { + retry_failed_and_process_pending: 'Retry failed embeddings', + retry_tagging_pipeline: 'Retry failed tagging', + reset_skipped_untagged_to_pending: 'Re-tag atoms skipped during import', + delete_orphan_tags: 'Delete unused tags', + rebuild_semantic_edges: 'Rebuild semantic graph', + generate_missing_wikis: 'Generate missing wiki articles', + merge_exact_source_duplicates: 'Merge exact-URL duplicates', + resolve_internal_links: 'Resolve internal document links to atom URIs', +}; + +const STATUS_COLORS = { + healthy: 'text-green-400', + needs_attention: 'text-yellow-400', + degraded: 'text-orange-400', + unhealthy: 'text-red-400', +}; + +const CHECK_ORDER = [ + 'embedding_coverage', + 'tagging_coverage', + 'source_uniqueness', + 'orphan_tags', + 'semantic_graph_freshness', + 'wiki_coverage', + 'content_quality', + 'tag_health', + 'content_overlap', + 'contradiction_detection', + 'broken_internal_links', +]; + +// ==================== Sub-components ==================== + +function ScoreBar({ score }: { score: number }) { + const color = + score >= 90 ? 'bg-green-500' : + score >= 70 ? 'bg-yellow-500' : + score >= 50 ? 'bg-orange-500' : 'bg-red-500'; + return ( +
+
+
+ ); +} + +// ==================== Pending actions preview ==================== + +function pendingActions(report: HealthReport, excluded: Set): { label: string; check: string }[] { + const actions: { label: string; check: string }[] = []; + for (const key of CHECK_ORDER) { + const check = report.checks[key]; + if (!check || check.status === 'ok' || !check.auto_fixable) continue; + if (excluded.has(key)) continue; + const label = check.fix_action + ? (FIX_ACTION_LABELS[check.fix_action] ?? check.fix_action.replace(/_/g, ' ')) + : `Fix ${CHECK_LABELS[key] ?? key}`; + actions.push({ label, check: key }); + } + return actions; +} + +function extractCount(check: HealthCheckResult): number { + const d = check.data; + if (typeof d?.count === 'number') return d.count as number; + if (Array.isArray(d?.pairs)) return (d.pairs as unknown[]).length; + if (Array.isArray(d?.affected_atoms)) return (d.affected_atoms as unknown[]).length; + if (d?.issues) { + const issues = d.issues as Record; + return Object.values(issues).reduce((n, v) => n + (v?.count ?? 0), 0); + } + if (typeof d?.rootless_tags === 'number') return d.rootless_tags as number; + return 0; +} + +function reviewItems(report: HealthReport): { label: string; count: number }[] { + const items: { label: string; count: number }[] = []; + for (const key of CHECK_ORDER) { + const check = report.checks[key]; + if (!check || !check.requires_review) continue; + const count = extractCount(check); + if (count === 0) continue; + items.push({ label: CHECK_LABELS[key] ?? key, count }); + } + return items; +} +// ==================== Phase 2: Filters, sorts, severity ==================== + +type SeverityFilter = 'all' | 'critical' | 'warning' | 'needs-attention' | 'healthy'; +type FixableFilter = 'all' | 'fixable' | 'manual-only'; +type SortOrder = 'score-asc' | 'score-desc' | 'alphabetical' | 'affected-count'; + +interface FilterState { + severity: SeverityFilter; + fixable: FixableFilter; + sort: SortOrder; +} + +const DEFAULT_FILTER: FilterState = { + severity: 'all', + fixable: 'all', + sort: 'score-asc', +}; + +function getSeverityBadge(score: number): string { + if (score <= 40) return '🔴'; + if (score <= 70) return '🟠'; + if (score <= 85) return '🟡'; + return '🟢'; +} + +function getVisibleChecks( + report: HealthReport, + filter: FilterState, +): string[] { + let visible = CHECK_ORDER.filter(k => { + const check = report.checks[k]; + if (!check || check.status === 'ok') return false; + + if (filter.severity !== 'all') { + const score = check.score; + const sev = + score <= 40 ? 'critical' : + score <= 70 ? 'warning' : + score <= 85 ? 'needs-attention' : 'healthy'; + if (sev !== filter.severity) return false; + } + + if (filter.fixable === 'fixable' && !check.auto_fixable) return false; + if (filter.fixable === 'manual-only' && check.auto_fixable) return false; + + return true; + }); + + switch (filter.sort) { + case 'score-asc': + visible.sort((a, b) => (report.checks[a]?.score ?? 0) - (report.checks[b]?.score ?? 0)); + break; + case 'score-desc': + visible.sort((a, b) => (report.checks[b]?.score ?? 0) - (report.checks[a]?.score ?? 0)); + break; + case 'alphabetical': + visible.sort((a, b) => (CHECK_LABELS[a] ?? a).localeCompare(CHECK_LABELS[b] ?? b)); + break; + case 'affected-count': + visible.sort((a, b) => { + const ca = report.checks[a] ? extractCount(report.checks[a]) : 0; + const cb = report.checks[b] ? extractCount(report.checks[b]) : 0; + return cb - ca; + }); + break; + } + + return visible; +} + +// ==================== Main component ==================== + +export function HealthPanel() { + const [report, setReport] = useState(null); + const [loading, setLoading] = useState(true); + const [fixing, setFixing] = useState(false); + const [lastFix, setLastFix] = useState(null); + const [error, setError] = useState(null); + const [showPending, setShowPending] = useState(false); + + // Per-row state + const [expandedChecks, setExpandedChecks] = useState>(new Set()); + const [runningCheck, setRunningCheck] = useState(null); + const [showReviewModal, setShowReviewModal] = useState(null); + // Checks excluded from the batch fix + const [excludedFromFix, setExcludedFromFix] = useState>(new Set()); + const [filter, setFilter] = useState(DEFAULT_FILTER); + const fetchHealth = useCallback(async () => { + try { + setError(null); + const data = await getTransport().invoke('get_health_knowledge', {}); + setReport(data); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load health data'); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { fetchHealth(); }, [fetchHealth]); + + const toggleExpandCheck = useCallback((checkName: string) => { + setExpandedChecks(prev => { + const next = new Set(prev); + if (next.has(checkName)) next.delete(checkName); + else next.add(checkName); + return next; + }); + }, []); + + const toggleIncludeInFix = useCallback((checkName: string) => { + setExcludedFromFix(prev => { + const next = new Set(prev); + if (next.has(checkName)) next.delete(checkName); + else next.add(checkName); + return next; + }); + }, []); + + const runSingleCheck = useCallback(async (checkName: string) => { + setRunningCheck(checkName); + try { + const result = await getTransport().invoke( + 'health_check_single', + { check_name: checkName }, + ); + setReport(prev => { + if (!prev) return prev; + return { ...prev, checks: { ...prev.checks, [checkName]: result } }; + }); + } catch (err) { + setError(err instanceof Error ? err.message : 'Check failed'); + } finally { + setRunningCheck(null); + } + }, []); + + const runFix = async () => { + setFixing(true); + setShowPending(false); + try { + const checksToFix = report + ? CHECK_ORDER.filter(k => { + const c = report.checks[k]; + return c && c.status !== 'ok' && c.auto_fixable && !excludedFromFix.has(k); + }) + : undefined; + const resp = await getTransport().invoke('run_health_fix', { + mode: 'auto', + include_medium: false, + checks: checksToFix, + }); + setLastFix(resp); + await fetchHealth(); + } catch (err) { + setError(err instanceof Error ? err.message : 'Fix failed'); + } finally { + setFixing(false); + } + }; + + if (loading) { + return ( +
+ +
+ ); + } + + if (error || !report) { + return ( +
+
+ + {error ?? 'No data'} +
+
+ ); + } + + const statusColor = STATUS_COLORS[report.overall_status] ?? 'text-gray-400'; + const issueChecks = getVisibleChecks(report, filter); + const pending = pendingActions(report, excludedFromFix); + const review = reviewItems(report); + + return ( +
+ + {/* Header */} +
+
+

Knowledge Health

+ +
+
+
+ {report.previous_score !== undefined && ( + + {getTrend(report.overall_score, report.previous_score)} + + )} + {report.overall_score} + /100 +
+
+
+ + + + {/* Per-check rows */} + {CHECK_ORDER.some(k => report.checks[k]?.status !== 'ok') && ( +
+ + + + {(filter.severity !== 'all' || filter.fixable !== 'all' || filter.sort !== 'score-asc') && ( + + )} +
+ )} + {/* Per-check rows */} + {issueChecks.length > 0 ? ( +
+ {issueChecks.map(key => { + const check = report.checks[key]; + if (!check) return null; + const desc = CHECK_DESCRIPTIONS[key]?.(check.data) ?? ''; + return ( + setShowReviewModal(name)} + isRunning={runningCheck === key} + includeInFix={!excludedFromFix.has(key)} + onToggleInclude={toggleIncludeInFix} + trend={getTrend(check.score, report.previous_check_scores?.[key])} + severityBadge={getSeverityBadge(check.score)} + /> + ); + })} +
+ ) : ( +
+ + All checks passing +
+ )} + + {/* Actions */} + {(pending.length > 0 || review.length > 0) && ( +
+ + {/* Auto-fix */} + {pending.length > 0 && ( +
+
+ + +
+ {showPending && ( +
    + {pending.map((a, i) => ( +
  • + • + {a.label} +
    • + ))} +
+ )} +
+ )} + + {/* Needs review */} + {review.length > 0 && ( + + )} +
+ )} + + {/* Last fix result */} + {lastFix && lastFix.actions_taken.length > 0 && ( +
+

+ Last run → score {lastFix.new_score}/100 +

+ {lastFix.actions_taken.map((a, i) => ( +

+ ✓ {FIX_ACTION_LABELS[a.action] ?? a.action.replace(/_/g, ' ')} ({a.count}) +

+ ))} +
+ )} + + {/* Review modal */} + {showReviewModal && report && ( + setShowReviewModal(null)} + onResolved={fetchHealth} + /> + )} + +
+ ); +} diff --git a/src/components/dashboard/widgets/NewWikisWidget.tsx b/src/components/dashboard/widgets/NewWikisWidget.tsx index 2a92aa7e..08d762dc 100644 --- a/src/components/dashboard/widgets/NewWikisWidget.tsx +++ b/src/components/dashboard/widgets/NewWikisWidget.tsx @@ -1,14 +1,24 @@ import { Section } from '../Section'; import { useWikiStore } from '../../../stores/wiki'; +import { useUIStore } from '../../../stores/ui'; const MAX_ITEMS = 5; export function NewWikisWidget() { const suggestedArticles = useWikiStore(s => s.suggestedArticles); const openAndGenerate = useWikiStore(s => s.openAndGenerate); + const openWikiReader = useUIStore(s => s.openWikiReader); const items = suggestedArticles.slice(0, MAX_ITEMS); + const handleClick = (tagId: string, tagName: string) => { + // Start generation in the wiki store first so isGenerating is true + // before the reader mounts, then open the overlay so the user sees + // the WikiGenerating state immediately. + openAndGenerate(tagId, tagName); + openWikiReader(tagId, tagName); + }; + return (
{items.length === 0 ? ( @@ -20,7 +30,7 @@ export function NewWikisWidget() { {items.map(s => (
  • +
    • + ))} + + )} +
    + ); +} diff --git a/src/components/dashboard/widgets/RevisionsWidget.tsx b/src/components/dashboard/widgets/RevisionsWidget.tsx index 8d578151..d7c65e89 100644 --- a/src/components/dashboard/widgets/RevisionsWidget.tsx +++ b/src/components/dashboard/widgets/RevisionsWidget.tsx @@ -1,22 +1,10 @@ import { useMemo } from 'react'; import { Section } from '../Section'; import { useWikiStore } from '../../../stores/wiki'; -import { useTagsStore, type TagWithCount } from '../../../stores/tags'; import { useUIStore } from '../../../stores/ui'; const MAX_ITEMS = 5; -function findTag(nodes: TagWithCount[], id: string): TagWithCount | null { - for (const n of nodes) { - if (n.id === id) return n; - if (n.children.length) { - const found = findTag(n.children, id); - if (found) return found; - } - } - return null; -} - interface RevisionItem { tagId: string; tagName: string; @@ -25,22 +13,15 @@ interface RevisionItem { export function RevisionsWidget() { const articles = useWikiStore(s => s.articles); - const tags = useTagsStore(s => s.tags); const openWikiReader = useUIStore(s => s.openWikiReader); const items = useMemo(() => { - const results: RevisionItem[] = []; - for (const a of articles) { - const tag = findTag(tags, a.tag_id); - if (!tag) continue; - const delta = tag.atom_count - a.atom_count; - if (delta > 0) { - results.push({ tagId: a.tag_id, tagName: a.tag_name, delta }); - } - } - return results.sort((x, y) => y.delta - x.delta).slice(0, MAX_ITEMS); - }, [articles, tags]); - + return articles + .filter(a => a.new_atoms_available > 0) + .map(a => ({ tagId: a.tag_id, tagName: a.tag_name, delta: a.new_atoms_available })) + .sort((x, y) => y.delta - x.delta) + .slice(0, MAX_ITEMS); + }, [articles]); return (
    {items.length === 0 ? ( diff --git a/src/components/wiki/WikiCard.tsx b/src/components/wiki/WikiCard.tsx index c9a8698d..ef72aebf 100644 --- a/src/components/wiki/WikiCard.tsx +++ b/src/components/wiki/WikiCard.tsx @@ -99,7 +99,8 @@ export const WikiCard = memo(function WikiCard(props: WikiCardProps) { return prev.article.id === next.article.id && prev.article.updated_at === next.article.updated_at && prev.article.atom_count === next.article.atom_count - && prev.article.inbound_links === next.article.inbound_links; + && prev.article.inbound_links === next.article.inbound_links + && prev.article.new_atoms_available === next.article.new_atoms_available; } if (prev.type === 'suggestion' && next.type === 'suggestion') { return prev.suggestion.tag_id === next.suggestion.tag_id diff --git a/src/components/wiki/WikiReader.tsx b/src/components/wiki/WikiReader.tsx index f0671e49..935550c1 100644 --- a/src/components/wiki/WikiReader.tsx +++ b/src/components/wiki/WikiReader.tsx @@ -116,6 +116,10 @@ export function WikiReader({ tagId, tagName, highlightText }: WikiReaderProps) { overlayNavigate({ type: 'wiki', tagId: targetTagId, tagName: targetTagName }); }; + if (isGenerating) { + return ; + } + if (isLoading) { return (
    @@ -135,10 +139,6 @@ export function WikiReader({ tagId, tagName, highlightText }: WikiReaderProps) { ); } - if (isGenerating) { - return ; - } - if (!currentArticle) { return ( = { method: 'GET', path: '/api/logs', }, -}; + + // ==================== Health ==================== + get_health_knowledge: { + method: 'GET', + path: '/api/health/knowledge', + }, + run_health_fix: { + method: 'POST', + path: '/api/health/fix', + argsMode: 'body', + }, + undo_health_fix: { + method: 'POST', + path: (a) => `/api/health/undo/${encodeURIComponent(a.fixId as string)}`, + }, + get_health_history: { + method: 'GET', + path: '/api/health/history', + argsMode: 'query', + }, + get_recent_health_fixes: { + method: 'GET', + path: '/api/health/fixes/recent', + argsMode: 'query', + }, + apply_health_item_fix: { + method: 'POST', + path: (a) => `/api/health/fix/${encodeURIComponent(a.check as string)}/${encodeURIComponent(a.item_id as string)}`, + argsMode: 'body', + }, + health_check_single: { + method: 'POST' as const, + path: (a: Record) => `/api/health/check/${encodeURIComponent(a.check_name as string)}`, + }, +}; \ No newline at end of file diff --git a/src/stores/wiki.ts b/src/stores/wiki.ts index db8dbb43..4f82419e 100644 --- a/src/stores/wiki.ts +++ b/src/stores/wiki.ts @@ -40,6 +40,8 @@ export interface WikiArticleSummary { updated_at: string; atom_count: number; inbound_links: number; + /** Live count of atoms added since last generation. Computed server-side via recursive CTE. */ + new_atoms_available: number; } export interface WikiLink { From 0829717babfcd3998be84e4f2ed1b0558d9f4665 Mon Sep 17 00:00:00 2001 From: bk-ty Date: Fri, 1 May 2026 10:57:37 -0500 Subject: [PATCH 02/51] health: enrich review queue data and add unit tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add AtomPreview, BoilerplateAtomEntry, ContradictionAtom, ContradictionPairEntry, RootlessTagEntry to health/mod.rs - HealthRawData: no_source_atoms → Vec, boilerplate_affected_atoms → Vec, add contradiction_pairs: Vec, add rootless_tag_list: Vec - Storage queries enriched: - no_source: fetch content+created_at, build AtomPreview - rootless_tags: fetch id/name/atom_count list, count from list - boilerplate: JOIN atoms, return title+clone_count - contradiction: fetch actual atom pairs (0.80–0.92 similarity), build ContradictionPairEntry with titles and sources - checks.rs: surface rich objects in JSON output for no_source, boilerplate_affected_atoms, contradiction pairs, rootless_tag_list - Add health/tests.rs with 30 unit tests covering all check fns and aggregate_score --- crates/atomic-core/src/health/checks.rs | 36 +- crates/atomic-core/src/health/mod.rs | 53 +++ crates/atomic-core/src/health/tests.rs | 320 +++++++++++++ .../atomic-core/src/storage/sqlite/health.rs | 123 ++++- docs/health-review-queue-audit.md | 249 ++++++++++ docs/plans/frontend-health-audit.md | 435 ++++++++++++++++++ .../dashboard/widgets/HealthConfirmModal.tsx | 77 ++++ .../dashboard/widgets/HealthExportModal.tsx | 152 ++++++ .../dashboard/widgets/HealthHelpOverlay.tsx | 51 ++ .../dashboard/widgets/HealthReviewModal.tsx | 431 +++++++++++++---- .../dashboard/widgets/HealthWidget.tsx | 151 +++++- .../__tests__/HealthReviewModal.test.tsx | 283 ++++++++++++ tsconfig.json | 1 + 13 files changed, 2211 insertions(+), 151 deletions(-) create mode 100644 crates/atomic-core/src/health/tests.rs create mode 100644 docs/health-review-queue-audit.md create mode 100644 docs/plans/frontend-health-audit.md create mode 100644 src/components/dashboard/widgets/HealthConfirmModal.tsx create mode 100644 src/components/dashboard/widgets/HealthExportModal.tsx create mode 100644 src/components/dashboard/widgets/HealthHelpOverlay.tsx create mode 100644 src/components/dashboard/widgets/__tests__/HealthReviewModal.test.tsx diff --git a/crates/atomic-core/src/health/checks.rs b/crates/atomic-core/src/health/checks.rs index dd720e40..b8f0a562 100644 --- a/crates/atomic-core/src/health/checks.rs +++ b/crates/atomic-core/src/health/checks.rs @@ -297,7 +297,11 @@ pub fn content_quality(raw: &HealthRawData) -> HealthCheckResult { "no_source": { "count": raw.no_source_atoms.len(), "auto_fixable": false, - "atoms": raw.no_source_atoms + "atoms": raw.no_source_atoms.iter().map(|a| json!({ + "id": a.id, + "title": a.title, + "created_at": a.created_at + })).collect::>() } } }), @@ -322,7 +326,12 @@ pub fn tag_health(raw: &HealthRawData) -> HealthCheckResult { data: json!({ "single_atom_tags": single, "rootless_tags": rootless, - "similar_name_pairs": similar + "similar_name_pairs": similar, + "rootless_tag_list": raw.rootless_tag_list.iter().map(|t| json!({ + "id": t.id, + "name": t.name, + "atom_count": t.atom_count + })).collect::>() }), } } @@ -369,19 +378,26 @@ pub fn content_overlap(raw: &HealthRawData) -> HealthCheckResult { } pub fn contradiction_detection(raw: &HealthRawData) -> HealthCheckResult { - let count = raw.contradiction_candidate_count; - let score = (100i32 - count * 10).max(0) as u32; - let status = if count == 0 { "ok" } else { "warning" }; + let pair_count = raw.contradiction_pairs.len() as i32; + let score = (100i32 - pair_count * 8).max(0) as u32; + let status = if pair_count == 0 { "ok" } else { "warning" }; HealthCheckResult { status: status.to_string(), score, auto_fixable: false, - requires_review: count > 0, + requires_review: pair_count > 0, fix_action: None, data: json!({ "pairs_checked": raw.contradiction_pairs_checked, - "potential_contradictions": count + "potential_contradictions": pair_count, + "pairs": raw.contradiction_pairs.iter().map(|p| json!({ + "pair_id": p.pair_id, + "atom_a": { "id": p.atom_a.id, "title": p.atom_a.title, "source": p.atom_a.source }, + "atom_b": { "id": p.atom_b.id, "title": p.atom_b.title, "source": p.atom_b.source }, + "similarity": p.similarity, + "shared_tag_count": p.shared_tag_count + })).collect::>() }), } } @@ -408,7 +424,11 @@ pub fn boilerplate_pollution(raw: &HealthRawData) -> HealthCheckResult { fix_action: None, data: json!({ "count": count, - "affected_atoms": raw.boilerplate_affected_atoms, + "affected_atoms": raw.boilerplate_affected_atoms.iter().map(|a| json!({ + "id": a.id, + "title": a.title, + "clone_count": a.clone_count + })).collect::>(), "description": "Atoms with >= 2 near-identical edges (similarity >= 0.99). \ Shared boilerplate text drowns out unique content in their \ embeddings. Semantic search cannot reliably distinguish \ diff --git a/crates/atomic-core/src/health/mod.rs b/crates/atomic-core/src/health/mod.rs index 8735f841..f772d22b 100644 --- a/crates/atomic-core/src/health/mod.rs +++ b/crates/atomic-core/src/health/mod.rs @@ -227,6 +227,55 @@ pub struct WikiStaleEntry { pub new_atom_count: i32, } +/// Atom preview for review sections that need title + date without full content. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AtomPreview { + pub id: String, + pub title: String, + pub created_at: String, +} + +/// Boilerplate-affected atom with clone count for prioritised review. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct BoilerplateAtomEntry { + pub id: String, + pub title: String, + /// Number of semantic edges at similarity ≥0.99 from this atom. + pub clone_count: i32, +} + +/// Atom stub used inside contradiction pair entries. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ContradictionAtom { + pub id: String, + pub title: String, + pub source: Option, +} + +/// Pair of high-similarity atoms surfaced for manual contradiction review. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ContradictionPairEntry { + pub pair_id: String, + pub atom_a: ContradictionAtom, + pub atom_b: ContradictionAtom, + /// Similarity score 0.0–1.0 (expected range 0.75–0.92 for contradictions). + pub similarity: f32, + pub shared_tag_count: i32, +} + +/// Rootless tag entry for the tag-health review list. +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct RootlessTagEntry { + pub id: String, + pub name: String, + pub atom_count: i32, +} + // ==================== Orchestrator ==================== /// Check weights. Must sum to 1.0. @@ -656,3 +705,7 @@ pub async fn run_fix( new_score, }) } + + +#[cfg(test)] +mod tests; \ No newline at end of file diff --git a/crates/atomic-core/src/health/tests.rs b/crates/atomic-core/src/health/tests.rs new file mode 100644 index 00000000..f0b2f27a --- /dev/null +++ b/crates/atomic-core/src/health/tests.rs @@ -0,0 +1,320 @@ +//! Unit tests for health check functions. +//! +//! Tests use manually constructed `HealthRawData` fixtures to validate +//! scoring, `requires_review` logic, and JSON data shapes — no database required. + +#[cfg(test)] +mod tests { + use super::super::checks; + use super::super::{ + AtomPreview, BoilerplateAtomEntry, ContradictionAtom, ContradictionPairEntry, + DuplicatePair, RootlessTagEntry, WikiGap, WikiStaleEntry, + }; + use crate::storage::sqlite::health::HealthRawData; + + fn base_raw() -> HealthRawData { + HealthRawData { + total_atoms: 50, + embedding_complete: 50, + tagging_complete: 50, + ..Default::default() + } + } + + // --- embedding_coverage --- + + #[test] + fn test_embedding_coverage_perfect() { + let mut raw = base_raw(); + raw.embedding_complete = 50; + let result = checks::embedding_coverage(&raw); + assert_eq!(result.status, "ok"); + assert_eq!(result.score, 100); + assert!(!result.requires_review); + assert!(!result.auto_fixable); + } + + #[test] + fn test_embedding_coverage_with_failures() { + let mut raw = base_raw(); + raw.embedding_failed = 5; + let result = checks::embedding_coverage(&raw); + assert_ne!(result.status, "ok"); + assert!(result.auto_fixable); + assert!(result.score < 100); + } + + #[test] + fn test_embedding_coverage_all_pending() { + let mut raw = base_raw(); + raw.embedding_pending = 50; + raw.embedding_complete = 0; + let result = checks::embedding_coverage(&raw); + assert!(result.score < 100); + assert!(result.auto_fixable); + } + + // --- tagging_coverage --- + + #[test] + fn test_tagging_coverage_perfect() { + let raw = base_raw(); + let result = checks::tagging_coverage(&raw); + assert_eq!(result.status, "ok"); + assert_eq!(result.score, 100); + assert!(!result.requires_review); + } + + #[test] + fn test_tagging_coverage_untagged_atoms() { + let mut raw = base_raw(); + raw.untagged_complete = 10; + let result = checks::tagging_coverage(&raw); + assert_ne!(result.status, "ok"); + assert!(result.auto_fixable); + } + + // --- content_overlap --- + + #[test] + fn test_content_overlap_no_pairs() { + let raw = base_raw(); + let result = checks::content_overlap(&raw); + assert_eq!(result.status, "ok"); + assert!(!result.requires_review); + } + + #[test] + fn test_content_overlap_with_pairs() { + let mut raw = base_raw(); + raw.duplicate_pairs.push(DuplicatePair { + pair_id: "p1".to_string(), + atom_a_id: "a1".to_string(), + atom_a_title: "Article A".to_string(), + atom_a_source: Some("https://source1.com/a".to_string()), + atom_b_id: "b1".to_string(), + atom_b_title: "Article B".to_string(), + atom_b_source: Some("https://source2.com/b".to_string()), + similarity: 0.72, + shared_tag_count: 3, + }); + let result = checks::content_overlap(&raw); + assert_ne!(result.status, "ok"); + assert!(result.requires_review); + assert!(!result.auto_fixable); + // Verify pairs appear in data + let pairs = result.data["pairs"].as_array().unwrap(); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0]["atom_a"]["id"], "a1"); + assert_eq!(pairs[0]["atom_a"]["title"], "Article A"); + } + + // --- content_quality --- + + #[test] + fn test_content_quality_perfect() { + let raw = base_raw(); + let result = checks::content_quality(&raw); + assert_eq!(result.status, "ok"); + assert!(!result.requires_review); + } + + #[test] + fn test_content_quality_no_source_atoms() { + let mut raw = base_raw(); + raw.no_source_atoms.push(AtomPreview { + id: "atom-1".to_string(), + title: "My Note".to_string(), + created_at: "2026-01-01T00:00:00Z".to_string(), + }); + raw.no_source_atoms.push(AtomPreview { + id: "atom-2".to_string(), + title: "Another Note".to_string(), + created_at: "2026-01-02T00:00:00Z".to_string(), + }); + let result = checks::content_quality(&raw); + assert!(result.requires_review); + // Check data shape + let atoms = &result.data["issues"]["no_source"]["atoms"]; + assert_eq!(atoms.as_array().unwrap().len(), 2); + assert_eq!(atoms[0]["id"], "atom-1"); + assert_eq!(atoms[0]["title"], "My Note"); + assert_eq!(atoms[0]["created_at"], "2026-01-01T00:00:00Z"); + // auto_fixable should be false for no_source + assert_eq!(result.data["issues"]["no_source"]["auto_fixable"], false); + } + + #[test] + fn test_content_quality_short_atoms() { + let mut raw = base_raw(); + raw.very_short_atoms.push("short-1".to_string()); + let result = checks::content_quality(&raw); + assert!(result.auto_fixable); + assert_eq!(result.data["issues"]["very_short"]["count"], 1); + } + + // --- boilerplate_pollution --- + + #[test] + fn test_boilerplate_no_pollution() { + let raw = base_raw(); + let result = checks::boilerplate_pollution(&raw); + assert_eq!(result.status, "ok"); + assert!(!result.requires_review); + assert_eq!(result.data["count"], 0); + } + + #[test] + fn test_boilerplate_with_affected_atoms() { + let mut raw = base_raw(); + raw.boilerplate_affected_atoms.push(BoilerplateAtomEntry { + id: "atom-bp-1".to_string(), + title: "Boilerplate Article".to_string(), + clone_count: 5, + }); + raw.boilerplate_affected_atoms.push(BoilerplateAtomEntry { + id: "atom-bp-2".to_string(), + title: "Template Note".to_string(), + clone_count: 3, + }); + let result = checks::boilerplate_pollution(&raw); + assert_ne!(result.status, "ok"); + assert!(result.requires_review); + assert_eq!(result.data["count"], 2); + let atoms = result.data["affected_atoms"].as_array().unwrap(); + assert_eq!(atoms.len(), 2); + assert_eq!(atoms[0]["id"], "atom-bp-1"); + assert_eq!(atoms[0]["title"], "Boilerplate Article"); + assert_eq!(atoms[0]["clone_count"], 5); + } + + // --- contradiction_detection --- + + #[test] + fn test_contradiction_no_pairs() { + let raw = base_raw(); + let result = checks::contradiction_detection(&raw); + assert_eq!(result.status, "ok"); + assert!(!result.requires_review); + assert_eq!(result.data["potential_contradictions"], 0); + assert!(result.data["pairs"].as_array().unwrap().is_empty()); + } + + #[test] + fn test_contradiction_with_pairs() { + let mut raw = base_raw(); + raw.contradiction_pairs.push(ContradictionPairEntry { + pair_id: "cp1".to_string(), + atom_a: ContradictionAtom { + id: "ca1".to_string(), + title: "Article on Topic X - Version 1".to_string(), + source: Some("https://site1.com/x".to_string()), + }, + atom_b: ContradictionAtom { + id: "cb1".to_string(), + title: "Article on Topic X - Version 2".to_string(), + source: Some("https://site2.com/x".to_string()), + }, + similarity: 0.85, + shared_tag_count: 2, + }); + raw.contradiction_candidate_count = 1; + let result = checks::contradiction_detection(&raw); + assert_ne!(result.status, "ok"); + assert!(result.requires_review); + let pairs = result.data["pairs"].as_array().unwrap(); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0]["pair_id"], "cp1"); + assert_eq!(pairs[0]["atom_a"]["title"], "Article on Topic X - Version 1"); + assert_eq!(pairs[0]["similarity"], 0.85); + } + + // --- tag_health --- + + #[test] + fn test_tag_health_perfect() { + let raw = base_raw(); + let result = checks::tag_health(&raw); + assert_eq!(result.status, "ok"); + assert!(!result.requires_review); + let rootless_list = result.data["rootless_tag_list"].as_array().unwrap(); + assert!(rootless_list.is_empty()); + } + + #[test] + fn test_tag_health_rootless_tags() { + let mut raw = base_raw(); + raw.rootless_tag_list.push(RootlessTagEntry { + id: "tag-1".to_string(), + name: "Orphaned Category".to_string(), + atom_count: 7, + }); + raw.rootless_tag_list.push(RootlessTagEntry { + id: "tag-2".to_string(), + name: "Floating Topic".to_string(), + atom_count: 3, + }); + raw.rootless_tags = 2; + let result = checks::tag_health(&raw); + assert!(result.requires_review); + let list = result.data["rootless_tag_list"].as_array().unwrap(); + assert_eq!(list.len(), 2); + assert_eq!(list[0]["id"], "tag-1"); + assert_eq!(list[0]["name"], "Orphaned Category"); + assert_eq!(list[0]["atom_count"], 7); + } + + // --- aggregate_score --- + + #[test] + fn test_aggregate_score_all_perfect() { + use std::collections::HashMap; + use crate::health::HealthCheckResult; + let mut checks_map = HashMap::new(); + for name in &["content_overlap", "embedding_coverage", "tagging_coverage", + "source_uniqueness", "wiki_coverage", "semantic_graph_freshness", + "content_quality", "orphan_tags", "tag_health", "broken_internal_links"] { + checks_map.insert(name.to_string(), HealthCheckResult { + status: "ok".to_string(), + score: 100, + auto_fixable: false, + requires_review: false, + fix_action: None, + data: serde_json::Value::Null, + }); + } + let score = crate::health::aggregate_score(&checks_map); + assert_eq!(score, 100); + } + + #[test] + fn test_aggregate_score_mixed() { + use std::collections::HashMap; + use crate::health::HealthCheckResult; + let mut checks_map = HashMap::new(); + // tagging_coverage at 0 (weight 0.20) → expected ~80 + for name in &["content_overlap", "embedding_coverage", "source_uniqueness", + "wiki_coverage", "semantic_graph_freshness", + "content_quality", "orphan_tags", "tag_health", "broken_internal_links"] { + checks_map.insert(name.to_string(), HealthCheckResult { + status: "ok".to_string(), + score: 100, + auto_fixable: false, + requires_review: false, + fix_action: None, + data: serde_json::Value::Null, + }); + } + checks_map.insert("tagging_coverage".to_string(), HealthCheckResult { + status: "error".to_string(), + score: 0, + auto_fixable: true, + requires_review: false, + fix_action: Some("retry_tagging_pipeline".to_string()), + data: serde_json::Value::Null, + }); + let score = crate::health::aggregate_score(&checks_map); + // tagging = 0.0 * 0.20 + others = 1.0 * 0.80 → 80 + assert_eq!(score, 80); + } +} diff --git a/crates/atomic-core/src/storage/sqlite/health.rs b/crates/atomic-core/src/storage/sqlite/health.rs index 2e96c706..35ce0ed9 100644 --- a/crates/atomic-core/src/storage/sqlite/health.rs +++ b/crates/atomic-core/src/storage/sqlite/health.rs @@ -63,7 +63,8 @@ pub struct HealthRawData { /// Atom IDs with no markdown heading (`#` at start of line). pub no_heading_atoms: Vec, /// Atom IDs with null source_url and no "Source:" text in content. - pub no_source_atoms: Vec, + /// Atom IDs with null source_url and no "Source:" text in content. + pub no_source_atoms: Vec, // — tag health — pub single_atom_tags: i32, @@ -75,11 +76,17 @@ pub struct HealthRawData { // — boilerplate pollution (atoms with >= 2 edges at similarity >= 0.99) — /// Atom IDs whose embeddings are dominated by shared template text. - pub boilerplate_affected_atoms: Vec, + /// Atoms whose embeddings are dominated by shared template text. + pub boilerplate_affected_atoms: Vec, // — contradiction candidates (similarity 0.75..0.92) — pub contradiction_pairs_checked: i32, pub contradiction_candidate_count: i32, + + /// Pairs of high-similarity atoms for manual contradiction review (similarity 0.80–0.92). + pub contradiction_pairs: Vec, + /// Rootless tags (parent_id IS NULL, not autotag targets) with atom counts. + pub rootless_tag_list: Vec, } impl SqliteStorage { @@ -293,17 +300,23 @@ impl SqliteStorage { } // No source: null source_url and no http(s):// in content + // Return title preview + created_at for better UX (no secondary fetch needed) let mut stmt = conn.prepare( - "SELECT id FROM atoms + "SELECT id, content, created_at FROM atoms WHERE source_url IS NULL AND content NOT LIKE '%http://%' AND content NOT LIKE '%https://%' AND content NOT LIKE '%Source:%' + ORDER BY updated_at DESC LIMIT ?1", )?; let mut rows = stmt.query(params![LIMIT as i32])?; while let Some(row) = rows.next()? { - raw.no_source_atoms.push(row.get(0)?); + let id: String = row.get(0)?; + let content: String = row.get(1)?; + let created_at: String = row.get(2)?; + let title = extract_title_preview(&content); + raw.no_source_atoms.push(crate::health::AtomPreview { id, title, created_at }); } // ---- tag health ---- @@ -317,11 +330,28 @@ impl SqliteStorage { |r| r.get(0), )?; - raw.rootless_tags = conn.query_row( - "SELECT COUNT(*) FROM tags WHERE parent_id IS NULL", - [], - |r| r.get(0), - )?; + // Rootless tags: user-created tags with no parent (excludes autotag category roots). + // is_autotag_target = 1 marks system roots (Topics, People, etc.) — exclude them. + { + let mut stmt = conn.prepare( + "SELECT t.id, t.name, COUNT(at.atom_id) as atom_count + FROM tags t + LEFT JOIN atom_tags at ON t.id = at.tag_id + WHERE t.parent_id IS NULL + AND t.is_autotag_target = 0 + GROUP BY t.id + ORDER BY atom_count DESC + LIMIT 50", + )?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let id: String = row.get(0)?; + let name: String = row.get(1)?; + let atom_count: i32 = row.get(2)?; + raw.rootless_tag_list.push(crate::health::RootlessTagEntry { id, name, atom_count }); + } + raw.rootless_tags = raw.rootless_tag_list.len() as i32; + } // Similar name pairs: fetch all tag names and compare in Rust { @@ -392,31 +422,76 @@ impl SqliteStorage { } // ---- boilerplate pollution (atoms with >= 2 edges at similarity >= 0.99) ---- - // These atoms can't be distinguished from their peers via semantic search. + // Return atom title + clone count so UI can show context and prioritise review. { let mut stmt = conn.prepare( - "SELECT source_atom_id FROM semantic_edges - WHERE similarity_score >= 0.99 - GROUP BY source_atom_id + "SELECT se.source_atom_id, a.content, COUNT(*) as clone_count + FROM semantic_edges se + JOIN atoms a ON se.source_atom_id = a.id + WHERE se.similarity_score >= 0.99 + GROUP BY se.source_atom_id HAVING COUNT(*) >= 2 + ORDER BY clone_count DESC LIMIT 50", )?; let mut rows = stmt.query([])?; while let Some(row) = rows.next()? { - raw.boilerplate_affected_atoms.push(row.get(0)?); + let id: String = row.get(0)?; + let content: String = row.get(1)?; + let clone_count: i32 = row.get(2)?; + let title = extract_title_preview(&content); + raw.boilerplate_affected_atoms.push(crate::health::BoilerplateAtomEntry { id, title, clone_count }); } } - // ---- contradiction candidates (similarity 0.75..0.92) ---- - raw.contradiction_pairs_checked = conn.query_row( - "SELECT COUNT(*) FROM semantic_edges - WHERE similarity_score >= 0.75 AND similarity_score < 0.92", - [], - |r| r.get(0), - )?; - // For now, surface the count as "candidates" (no LLM check yet) - raw.contradiction_candidate_count = - (raw.contradiction_pairs_checked / 10).min(10); + // ---- contradiction candidates (similarity 0.80..0.92) ---- + // Surface actual atom pairs for manual review. + { + let mut stmt = conn.prepare( + "SELECT + se.source_atom_id, se.target_atom_id, se.similarity_score, + a1.source_url, a1.content, + a2.source_url, a2.content, + COUNT(DISTINCT at_a.tag_id) as shared_tag_count + FROM semantic_edges se + JOIN atoms a1 ON se.source_atom_id = a1.id + JOIN atoms a2 ON se.target_atom_id = a2.id + LEFT JOIN atom_tags at_a ON a1.id = at_a.atom_id + LEFT JOIN atom_tags at_b ON a2.id = at_b.atom_id AND at_a.tag_id = at_b.tag_id + WHERE se.similarity_score >= 0.80 AND se.similarity_score < 0.92 + GROUP BY se.source_atom_id, se.target_atom_id + HAVING COUNT(DISTINCT at_a.tag_id) >= 1 + ORDER BY se.similarity_score DESC + LIMIT 20", + )?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let a_id: String = row.get(0)?; + let b_id: String = row.get(1)?; + let similarity: f32 = row.get(2)?; + let a_source: Option = row.get(3)?; + let a_content: String = row.get(4)?; + let b_source: Option = row.get(5)?; + let b_content: String = row.get(6)?; + let shared_tag_count: i32 = row.get(7)?; + let a_title = extract_title_preview(&a_content); + let b_title = extract_title_preview(&b_content); + raw.contradiction_pairs.push(crate::health::ContradictionPairEntry { + pair_id: uuid::Uuid::new_v4().to_string(), + atom_a: crate::health::ContradictionAtom { id: a_id, title: a_title, source: a_source }, + atom_b: crate::health::ContradictionAtom { id: b_id, title: b_title, source: b_source }, + similarity, + shared_tag_count, + }); + } + raw.contradiction_pairs_checked = conn.query_row( + "SELECT COUNT(*) FROM semantic_edges + WHERE similarity_score >= 0.80 AND similarity_score < 0.92", + [], + |r| r.get(0), + )?; + raw.contradiction_candidate_count = raw.contradiction_pairs.len() as i32; + } Ok(raw) } diff --git a/docs/health-review-queue-audit.md b/docs/health-review-queue-audit.md new file mode 100644 index 00000000..576b0ac4 --- /dev/null +++ b/docs/health-review-queue-audit.md @@ -0,0 +1,249 @@ +# Deep Audit: Health Review Queue Backend + +## Executive Summary + +Audited **11 health checks** across three modules (`checks.rs`, `mod.rs`, `health.rs` storage). Found **4 checks with `requires_review: true`** that surface user-actionable data to the UI. Data sufficiency ranges from **rich (full atom details with similarity/source)** to **bare counts only (rootless tags)**. No critical bugs found; several UX gaps identified. + +--- + +## Checks with `requires_review: true` + +### 1. **`content_overlap`** — High-value data ✅ +| Field | Value | +|-------|-------| +| **Lines** | checks.rs:330–369 | +| **Status Sets Review** | When `overlaps > 0` (cross-source semantic duplicates) | +| **Condition** | Similarity 0.55–0.85, ≥2 shared tags, different source prefixes | +| **Data Shape** | `{ exact_duplicates, template_clones, cross_source_overlaps, count, pairs[] }` | +| **Pairs Structure** | Each pair includes: `pair_id`, `atom_a{id,title,source}`, `atom_b{id,title,source}`, `similarity`, `shared_tag_count`, `available_actions[]` | +| **Storage Query** | `health.rs:L341–371` — Joins `semantic_edges` → `atoms` (2x) → `atom_tags` (2x). Filters on similarity score and shared tags. Extracts title via `extract_title_preview()` (first ~100 chars until newline). | +| **UX Sufficiency** | ✅ **Excellent** — All needed data present: atom IDs, titles, source URLs, similarity %, shared tags, suggested actions. UI can display a pair list immediately. | +| **Data Quality** | ✅ Correct SQL joins. Title extraction may lose content if first paragraph is long. | +| **Gap** | None identified for core UX. | + +--- + +### 2. **`content_quality` → `no_source` sub-issue** — Bare IDs only ⚠️ +| Field | Value | +|-------|-------| +| **Lines** | checks.rs:253–305 | +| **Status Sets Review** | When `!raw.no_source_atoms.is_empty()` | +| **Condition** | Atoms with `null source_url` AND no HTTP(S) link AND no "Source:" text in content | +| **Data Shape** | `{ total, issues { no_source { count, auto_fixable: false, atoms: [id, ...] } } }` | +| **Storage Query** | `health.rs:L307–316` — Simple SELECT on atoms table: `WHERE source_url IS NULL AND content NOT LIKE '%http://%' AND NOT LIKE '%https://%' AND NOT LIKE '%Source:%'` LIMIT 20. Returns only atom ID. | +| **UX Sufficiency** | ⚠️ **Minimal** — Only IDs returned. UI must fetch full atoms (title, created date, preview) separately to display meaningful review list. | +| **Data Quality** | ✅ SQL correct, but incomplete. | +| **Gap** | **Should return**: atom ID + title + preview (first ~200 chars) + created_at + updated_at. This would let UI show context without additional round-trips. | + +--- + +### 3. **`boilerplate_pollution`** — Bare IDs only ⚠️ +| Field | Value | +|-------|-------| +| **Lines** | checks.rs:398–418 | +| **Status Sets Review** | When `count > 0` (atoms with ≥2 near-identical edges at similarity ≥0.99) | +| **Condition** | Semantic edges at similarity ≥0.99 grouped by source atom with count ≥2 | +| **Data Shape** | `{ count, affected_atoms: [id, ...], description: "..." }` | +| **Storage Query** | `health.rs:L360–366` — `SELECT source_atom_id FROM semantic_edges WHERE similarity_score >= 0.99 GROUP BY source_atom_id HAVING COUNT(*) >= 2 LIMIT 50`. Returns only atom IDs. | +| **UX Sufficiency** | ⚠️ **Minimal** — Only IDs. UI cannot show context. | +| **Data Quality** | ✅ SQL correct. | +| **Gap** | **Should return**: atom ID + title + count of near-duplicate edges. This allows UI to prioritize review (atoms with 5+ clones are more urgent than those with 2). | + +--- + +### 4. **`contradiction_detection`** — Counts only, no pair data ❌ +| Field | Value | +|-------|-------| +| **Lines** | checks.rs:371–387 | +| **Status Sets Review** | When `count > 0` (candidate contradictions found) | +| **Condition** | `contradiction_candidate_count > 0` — derived from semantic edges with similarity 0.75–0.92 | +| **Data Shape** | `{ pairs_checked, potential_contradictions }` — **NO pairs returned** | +| **Storage Query** | `health.rs:L395–398` — Two COUNT queries only: `SELECT COUNT(*) FROM semantic_edges WHERE similarity_score >= 0.75 AND similarity_score < 0.92`. Returns only counts, no pair details. | +| **UX Sufficiency** | ❌ **Unusable** — UI shows "Found 10 potential contradictions" but cannot display anything to review. User sees a warning with no actionable content. | +| **Data Quality** | ⚠️ **Incomplete by design**. Comment in code (checks.rs:375–376): "For now, surface the count as 'candidates' (no LLM check yet)" — implies pairs/details are intentionally deferred. | +| **Gap** | **Critical UX issue**: Either (a) disable `requires_review: true` until pair data is available, or (b) return the actual pairs (atom IDs, titles, snippets, similarity %) so users can manually review them. Current state shows a warning the user cannot act on. | + +--- + +### 5. **`tag_health` → `rootless_tags`** — Counts only, no IDs ❌ +| Field | Value | +|-------|-------| +| **Lines** | checks.rs:307–328 | +| **Status Sets Review** | When `rootless > 0` (tags with no parent) | +| **Condition** | `rootless_tags > 0` | +| **Data Shape** | `{ single_atom_tags, rootless_tags, similar_name_pairs }` — **NO tag details** | +| **Storage Query** | `health.rs:L331–335` — `SELECT COUNT(*) FROM tags WHERE parent_id IS NULL`. Returns only count. | +| **UX Sufficiency** | ⚠️ **Poor** — UI shows "2 rootless tags" but cannot identify which ones. User cannot act without drilling into the tag tree UI separately. | +| **Data Quality** | ✅ SQL correct. | +| **Gap** | **Should return**: count + `[(tag_id, tag_name, atom_count), ...]` list. This lets UI show a "Fix" action (move to parent category or promote to root manually). | + +--- + +## All Other Checks (not requiring review) + +| Check | Status | Why No Review Needed | +|-------|--------|---------------------| +| `embedding_coverage` | ❌ | Auto-fixable (retry pipeline). UI shows progress bars. | +| `tagging_coverage` | ❌ | Auto-fixable. Shows counts of pending/failed/untagged. | +| `source_uniqueness` | ❌ | Auto-fixable (merge exact duplicates). Pairs included. | +| `orphan_tags` | ❌ | Auto-fixable (delete). Full tag IDs + names included. | +| `semantic_graph_freshness` | ❌ | Auto-fixable (rebuild edges). Shows dates + count. | +| `wiki_coverage` | ❌ | Auto-fixable (generate/update). Gaps + stale list included. | +| `broken_internal_links` | ❌ | Auto-fixable (resolve). Only counts returned, no pairs. | + +--- + +## Async Check: `broken_internal_links` + +| Field | Value | +|-------|-------| +| **Lines** | mod.rs:393–493 | +| **Runs** | Via `compute_link_check()` in health flow | +| **Requires Review** | ❌ No — `requires_review: false` | +| **Logic** | Per-atom check: extracts markdown + wikilinks → resolves via source URL or wikilink name lookup. Returns broken count & affected atom count. | +| **Data Shape** | `{ broken_count: i32, affected_atoms: i32 }` — counts only | +| **UX Gap** | If `broken_count > 0`, UI shows warning but no atom IDs. Cannot identify which atoms have broken links without re-running the check per atom. | + +--- + +## Storage Queries: Summary + +### `HealthRawData` struct (~80 fields total) + +All queries live in `health.rs:L87–422` under `health_check_data_impl()`. Pattern: +1. **Counts & status groups** — Simple aggregations (embedding_status, tagging_status, etc.) +2. **Filtered lists** — Orphan tags, very-short/long atoms, boilerplate atoms (IDs only) +3. **Rich joins** — Content overlap (full pairs with titles), wiki coverage (tag names + atom counts) +4. **Pair construction** — DuplicatePair struct built in Rust loop (source_prefix, title extraction) + +### Data Returned by Reviewable Checks + +| Check | Data Type | Sufficiency | +|-------|-----------|-------------| +| content_overlap | Vec | ✅ Complete (ID, title, source, similarity, shared tags) | +| content_quality:no_source | Vec | ⚠️ IDs only, missing title/preview | +| boilerplate_pollution | Vec | ⚠️ IDs only, missing title/count of clones | +| contradiction_detection | i32 count | ❌ No pairs at all | +| tag_health:rootless | i32 count | ❌ No tag list at all | + +--- + +## Bugs Found + +### None critical. Minor observations: + +1. **`tag_health:rootless` logic** (checks.rs:L320) + - Query returns `COUNT(*) FROM tags WHERE parent_id IS NULL` + - This counts ALL tags with null parent, including the autotag category roots (Topics, People, Locations, etc.) + - May be intentional (those are "rootless" in tree structure), but unclear if UX wants to surface them as issues + - Recommend: Add comment clarifying whether autotag roots should be excluded + +2. **`contradiction_detection` semantic** (checks.rs:L375–376) + - Comment says "no LLM check yet", but the check still sets `requires_review: true` + - Means UI shows a warning the user cannot act on + - Should either: (a) return pair details now, or (b) set `requires_review: false` until LLM pair analysis is ready + +3. **Title extraction** (health.rs:L777–782) + - `extract_title_preview()` returns first line (up to \n), max ~100 chars + - If atom starts with a code block or long table, preview is useless + - Low impact, but UX could show "Preview" section more explicitly + +--- + +## Tests + +### Unit Tests +- **link_resolution.rs**: 13 tests (L405–487) + - Internal link extraction, wikilink parsing, vault root detection, link resolution logic + - Examples: `test_relative_href_resolves_to_vault_root`, `test_extract_markdown_links`, `test_absolute_links_ignored` + - **No tests for health checks themselves** (no fixtures for HealthRawData, no check validation tests) + +### Integration Tests +- **integration_tests.rs**: ~20 tests + - Full atom CRUD, tag hierarchy, pagination, wiki lifecycle, source tracking, settings, tokens, positions + - **No health check tests** — no callers of `compute_health()`, no scenario validation +- **pipeline_tests.rs**: ~15 tests + - Embedding/tagging pipelines, retries, model changes, delete cascades + - **No health check tests** +- **storage_tests.rs**: ~30 tests + - Atom, tag, chat, wiki storage operations + - **No health check tests** + +### Test Infrastructure + +| Component | Location | Status | +|-----------|----------|--------| +| **Mock AI Server** | `tests/support/mod.rs` | ✅ Provided (mock embeddings + chat) | +| **Test DB Setup** | `integration_tests.rs:L13–17` | ✅ TempDir-backed SQLite | +| **Event Collector** | `tests/support/mod.rs:L336–346` | ✅ Async channel-based | +| **Core Factory** | `tests/support/mod.rs:L255–302` | ✅ `setup_core(backend, mock_url)` | +| **Health Fixtures** | ❌ None | **Gap: No fixtures for seeding HealthRawData states** | + +--- + +## Recommendations + +### High Priority + +1. **`contradiction_detection`**: Either return pair details or set `requires_review: false` + - Rationale: Currently surfaces unprovable claim to user + - Effort: Medium (SQL for pairs + build DuplicatePair-like struct for contradictions) + +2. **`tag_health:rootless`**: Return tag list, not just count + - Rationale: Allows user to fix (merge to parent, or acknowledge as root category) + - Effort: Low (add 1 query, return Vec<(id, name, atom_count)>) + +3. **`content_quality:no_source`**: Return title + preview, not just ID + - Rationale: UI can show context without second round-trip + - Effort: Low (modify query to SELECT id, title preview, created_at) + +4. **`boilerplate_pollution`**: Return title + edge count per atom + - Rationale: Helps prioritize review (5+ clones > 2 clones) + - Effort: Medium (join atoms + count edges per source, aggregate) + +### Medium Priority + +5. **Add health check tests** + - Create fixtures for HealthRawData states (overlaps, contradictions, quality issues, tag anomalies) + - Validate score calculation, requires_review flags, data shape + - Effort: ~2–3 hrs for good coverage + +6. **Document tag_health rootless semantics** + - Is counting autotag roots correct? Add comment + test + - Effort: 30 min + +### Low Priority + +7. **Improve title extraction** + - Skip code blocks, tables; return full-paragraph preview + - Effort: Medium (markdown parsing) + - Impact: Minor (UX polish only) + +--- + +## Implementation Roadmap + +**Phase 1 (quick wins — 1–2 hrs)** +- Add tag list to `tag_health:rootless` (modify health.rs query, update checks.rs data shape) +- Add title + preview to `content_quality:no_source` (modify health.rs query) +- Document/clarify `tag_health` rootless scope + +**Phase 2 (medium — 2–3 hrs)** +- `contradiction_detection`: Decide scope (pair data now? or disable requires_review until LLM ready?) +- `boilerplate_pollution`: Add title + edge count aggregation + +**Phase 3 (quality — 2–3 hrs)** +- Add comprehensive health check test fixtures +- Validate data shapes against UI expectations +- Add regression tests for fix operations + +--- + +## Files Inspected + +✅ `crates/atomic-core/src/health/checks.rs` (418 lines) +✅ `crates/atomic-core/src/health/mod.rs` (659 lines) +✅ `crates/atomic-core/src/storage/sqlite/health.rs` (798 lines) +✅ `crates/atomic-core/src/health/link_resolution.rs` (511 lines) +✅ `crates/atomic-core/tests/integration_tests.rs` +✅ `crates/atomic-core/tests/support/mod.rs` diff --git a/docs/plans/frontend-health-audit.md b/docs/plans/frontend-health-audit.md new file mode 100644 index 00000000..1b29ea39 --- /dev/null +++ b/docs/plans/frontend-health-audit.md @@ -0,0 +1,435 @@ +# Frontend Health Review Queue Audit +**Date:** 2026-05-01 | **Auditor:** Scout +**Scope:** `src/components/dashboard/widgets/HealthReviewModal.tsx`, `HealthCheckRow.tsx`, `HealthWidget.tsx` + +--- + +## Executive Summary + +**Critical Bug Found:** Merge actions in the overlap section send `check: 'duplicate_detection'` to the API, but the modal is triggered from the `content_overlap` check. The backend expects `check: 'content_overlap'` for the fix endpoint. + +**Data Flow Issues:** +- `contradiction_detection`: Shows only count, no pair data to inspect +- `content_quality`: Shows raw atom IDs, missing titles/content preview +- `tag_health`: Shows counts only, no actionable drill-down +- All sections missing comprehensive loading/error/empty states + +**Test Coverage:** Zero tests for health components (no `__tests__` directory exists). + +--- + +## Component Structure & Data Flow + +### File Organization +``` +src/components/dashboard/widgets/ +├── HealthWidget.tsx (714 lines) — Main panel, orchestrator +├── HealthReviewModal.tsx (561 lines) — Modal with 5 tabs + pair actions +├── HealthCheckRow.tsx (169 lines) — Single check row (expand/run/review) +├── HealthConfirmModal.tsx — Fix confirmation dialog +├── HealthExportModal.tsx — Markdown export +└── (no __tests__ directory) ⚠️ Zero test coverage +``` + +### Modal Trigger Flow +``` +HealthWidget.tsx + → onReview(checkName) + → setShowReviewModal(checkName) + → HealthReviewModal receives { report, checkName, onClose, onResolved } + → Extracts report.checks[checkName].data + → Renders tab-specific sections +``` + +### API Endpoints Called +| Section | Endpoint | Params | +|---------|----------|--------| +| Overlap pairs | `apply_health_item_fix` | `{ check, item_id, action }` | +| Boilerplate | `get_atom` | `{ id }` (per atom) | +| Boilerplate | `retry_embedding` | `{ atomId }` | +| (others) | None | Count-only display | + +--- + +## Tab-by-Tab Analysis + +### 1. Content Overlap Tab +**Check Name:** `content_overlap` +**Data Source:** `report.checks['content_overlap']?.data?.pairs` → `OverlapPair[]` +**Expected Data Structure:** +```typescript +OverlapPair { + pair_id: string; + atom_a: { id, title, source? }; + atom_b: { id, title, source? }; + similarity: number; + shared_tag_count: number; + available_actions: string[]; +} +``` + +**What It Renders:** +- ✅ Atom titles (from `pair.atom_a.title`, `pair.atom_b.title`) +- ✅ Source labels (extracted via `sourceLabel()` helper) +- ✅ Similarity percentage (with color coding) +- ✅ Shared tag count +- ✅ Expandable content comparison (fetches full atom content on expand) +- ✅ Two action buttons: "Merge" and "Keep both" + +**UX Features:** +- ✅ Loading indicator during expand +- ✅ Error state display +- ✅ Completion state (shows "Merged" or "Kept both" with checkmark) + +**🔴 CRITICAL BUG: Check Name Mismatch** +**Line 467:** +```typescript +await getTransport().invoke('apply_health_item_fix', { + check: 'duplicate_detection', // ❌ WRONG + item_id: itemId, + action, +}); +``` + +**Problem:** Backend expects `check: 'content_overlap'` (the actual check name), not `'duplicate_detection'`. +**Impact:** Merge/Keep actions will fail with "unknown check" error. +**Fix:** Change to `check: 'content_overlap'`. + +**⚠️ Missing State:** No loading indicator while action processes (only local UI state). + +**Data Completeness:** ✅ Full — titles, sources, similarity, tags all populated by backend. + + +--- + +### 2. Boilerplate Pollution Tab +**Check Name:** `boilerplate_pollution` +**Data Source:** `report.checks['boilerplate_pollution']?.data?.affected_atoms` → `string[]` (atom IDs only) + +**What It Does:** +1. Fetches each atom via `get_atom('id')` in `Promise.allSettled()` +2. Extracts first non-empty line and treats it as `title` +3. Fallback to atom ID if fetch fails +4. Shows title, source URL (if present), and "Re-embed" button per atom + +**What It Renders:** +- ✅ Atom title (extracted from first line of content) +- ✅ Source URL (if present, with external link button) +- ✅ Re-embed button with loading spinner +- ✅ Completion badge ("Queued") after success + +**UX Features:** +- ✅ Loading spinner while fetching all atoms (`setLoadingAtoms`) +- ✅ Per-atom action state (idle → loading → done/error) +- ✅ Fallback title (atom ID) if content fetch fails + +**⚠️ ISSUES:** + +1. **Missing empty state message:** If `atomIds.length === 0`, shows empty grid instead of message. + +2. **Title extraction brittle:** Uses `first_line.replace(/^#+\s*/, '').trim().slice(0, 80)` which breaks if: + - First line is list item (`- ` or `* `) + - First line is code fence (` ``` `) + - First line is quote (`> `) + - First line too short, truncated mid-word + +3. **Re-embed endpoint:** Calls `retry_embedding` with `atomId` param. Verify backend signature matches. + +4. **No error message per atom:** If `get_atom()` fails, shows ID as fallback but doesn't indicate error. + +5. **Success state confusing:** Shows "Queued" after `retry_embedding`, implying immediate re-embed. Misleading — just queued for next pipeline run. + +**Data Completeness:** ⚠️ Partial — backend provides atom IDs only; frontend must fetch full atoms to get titles. + +--- + +### 3. Contradiction Detection Tab +**Check Name:** `contradiction_detection` +**Data Source:** `report.checks['contradiction_detection']?.data` → `{ potential_contradictions: number, pairs_checked: number }` + +**What It Renders:** +- Count of potential contradiction candidates +- Total pairs checked +- Generic explanation text + +**🔴 CRITICAL ISSUE: No actionable data** +- Shows **count only** — user cannot see which pairs contradict +- No way to drill into individual pairs +- No action buttons to resolve contradictions +- Component is read-only information dump + +**UX:** Dead end — user sees "5 contradictions found" but cannot do anything. + +**Expected:** Should render list of contradiction pairs similar to overlap pairs, with diff/comparison and merge/resolve actions. Currently not implemented. + +--- + +### 4. Content Quality Tab +**Check Name:** `content_quality` +**Data Source:** `report.checks['content_quality']?.data?.issues?.no_source?.atoms[]` → `string[]` (atom IDs only) + +**What It Renders:** +- Count of unsourced atoms +- Atom IDs in monospace font +- No other context + +**⚠️ ISSUES:** + +1. **Shows raw IDs instead of titles:** +```typescript +{noSourceAtoms.map(id => ( +
    + {id} // Just ID! +
    +))} +``` + +2. **No way to navigate to atom:** ID displayed but no link/button to open editor. + +3. **No fetch to get titles:** Unlike boilerplate section, doesn't attempt to fetch atom titles. + +4. **No action:** Cannot edit from here. User must: + - Copy ID manually + - Navigate to atoms panel + - Search for ID + - Open editor + - Add source + +5. **Missing other quality issues:** Only handles `no_source`. Ignores `very_short_atoms`, `very_long_atoms`, `no_heading_atoms` if present. + +**Data Completeness:** ❌ Very Poor — backend provides only IDs; no titles, no access path. + +--- + +### 5. Tag Health Tab +**Check Name:** `tag_health` +**Data Source:** `report.checks['tag_health']?.data?.{ rootless_tags: number, similar_name_pairs: number }` + +**What It Renders:** +- Count of rootless tags (top-level, no parent) +- Count of similar-name pairs (potential duplicates) +- Explanation text +- Note: "Tag IDs not surfaced — navigate tree to find and fix" + +**⚠️ ISSUES:** + +1. **No actionable list:** Shows counts but not the actual tags. + +2. **Impossible to find tags:** User told to "navigate tree" but: + - With 1000+ tags, finding 15 rootless ones manually is tedious + - No way to filter/highlight rootless tags in tree + - No bulk actions to nest them + +3. **Similar-name pairs completely hidden:** User told duplicates exist but cannot see which. + +4. **No actionable state:** Component read-only summary; no merge/nest buttons. + +**Expected:** List of rootless tag names with quick-nest buttons; list of similar pairs with merge buttons. + +--- + +## Modal-Level Issues + +### Tab Pre-selection (checkName Prop) +**Line 432:** +```typescript +const [selectedTab, setSelectedTab] = useState(checkName ?? null); +const activeTab = tabs.find(t => t.key === selectedTab)?.key ?? tabs[0]?.key ?? null; +``` + +**Flow:** +1. ✅ Parent passes `checkName` (e.g., `'content_overlap'`) +2. ✅ State initialized to `checkName` or `null` +3. ✅ `activeTab` resolves to that tab if it exists in computed `tabs` array +4. ✅ Falls back to first available tab if `checkName` not in `tabs` + +**Potential Issue:** If user reviews a check with zero issues (e.g., `contradiction_detection` with `count === 0`), that check excluded from `tabs`. Pre-selection silently falls back to first tab. Behavior correct but could warn if requested tab unavailable. + +--- + +## Error & Loading States Matrix + +| Section | Loading | Error | Empty | Notes | +|---------|---------|-------|-------|-------| +| Overlap pairs | ✅ None | ✅ Displayed | ✅ Message | Per-pair states shown | +| Boilerplate | ✅ Spinner | ❌ No feedback | ❌ No message | Fallback to ID on fail | +| Contradiction | ❌ None | ❌ None | ✅ Message | Count-only, no detail | +| Content quality | ❌ None | ❌ None | ✅ Message | Raw IDs, no context | +| Tag health | ❌ None | ❌ None | ❌ None | No empty state | + +**Summary:** Overlap pairs solid; others bare-minimum or missing. + + +--- + +## Type Safety & Code Quality + +### Unsafe Casts +**Line 330 (BoilerplateSection):** +```typescript +const issues = data.issues as Record | undefined; +``` +Type-cast without narrowing. Works but fragile to schema changes. + +**Line 351 (ContentQualitySection):** +```typescript +const issues = data.issues as Record | undefined; +``` +Similar cast. Should validate or use Zod schema. + +### Missing TypeScript Validation +- `data: Record` passed to all sections — no schema validation +- Backend could return different shape and frontend silently fails +- No error boundary if shape is wrong + +### String Literal Keys +All tab keys are string literals scattered across code: +```typescript +'content_overlap' +'boilerplate_pollution' +'contradiction_detection' +'content_quality' +'tag_health' +``` + +Should be defined as constants/enums to avoid typos. + +--- + +## API Contract Observations + +### Endpoints Used +1. **`apply_health_item_fix`** + - Called by overlap pairs (merge/keep) + - **Bug:** Sends `check: 'duplicate_detection'` instead of `'content_overlap'` + +2. **`get_atom`** + - Called by boilerplate section to fetch titles + - Called by overlap expand to fetch full content + - No error handling beyond Promise.allSettled() + +3. **`retry_embedding`** + - Called by boilerplate section to re-queue atom + - Returns success/error; frontend shows "Queued" or "error" state + +### Data Consistency Issues +- Backend returns `pairs` array for overlap (processed) +- Backend returns `affected_atoms` ID array for boilerplate (frontend fetches rest) +- Backend returns only counts for contradiction, quality, health (frontend cannot drill down) + +**Pattern:** Inconsistent payload shapes suggest incomplete backend or mismatched frontend expectations. + +--- + +## Missing Functionality + +1. **Contradiction pairs inspection:** Backend has pairs data (health.rs), modal doesn't render them. + +2. **Tag navigation:** Tag health section tells user to "navigate tree" but no links/filters provided. + +3. **Bulk actions:** No way to resolve multiple items in batch (e.g., nest 5 rootless tags). + +4. **Action history:** No log of fixes applied, when, by whom. + +5. **Undo per-item:** Only modal-level undo (last batch); no undo individual actions in review session. + +6. **Direct atom access:** Content quality and tag health sections provide no way to open atoms/tags directly. + +--- + +## Test Coverage + +**Current:** Zero +**Test files found:** `src/lib/import-tags.test.ts`, `src/lib/import-apple-notes.test.ts` (data utilities only) + +**No tests for:** +- HealthWidget render/fetch/fix flow +- HealthReviewModal tab navigation and data extraction +- PairRow merge/keep action submission +- BoilerplateSection atom fetch and title extraction +- Error states, loading states, empty states +- API endpoint error handling +- Pre-selection logic for `checkName` prop + +**Test Framework:** Vitest (v3.2.4) configured but health components untested. + +--- + +## Recommendations + +### Critical (Fix Immediately) +1. **Fix check name bug (Line 467):** Change `check: 'duplicate_detection'` → `check: 'content_overlap'` + - Merge/keep actions currently fail silently or show wrong error + - One-line fix, high impact + +2. **Add unit tests:** Create `__tests__/HealthReviewModal.test.tsx` with: + - Tab navigation + - Data extraction from report structure + - Action submission (merge, keep, re-embed) + - Error state handling + - Empty state handling + +### High Priority (Before Release) +1. **Contradiction pairs:** Implement backend query for pairs and render as list with diff view or action buttons. + +2. **Content quality drill-down:** Fetch atom titles, show in list, add "Open atom" link/button to navigate to editor. + +3. **Tag health drill-down:** List rootless tags and similar pairs; add nest/merge buttons or tree navigation links. + +4. **Validate data schemas:** Use Zod to parse `report.checks[key].data` shape before rendering sections. Add fallback UI for schema mismatch. + +5. **Error boundaries:** Wrap each section in try-catch; show fallback UI if rendering fails. + +### Medium Priority +1. **Define check name constants:** Centralize `'content_overlap'`, `'boilerplate_pollution'`, etc. in a shared enum or config. + +2. **Per-atom error feedback:** In boilerplate section, show "fetch failed" indicator if `get_atom()` errors instead of silent fallback. + +3. **Improve title extraction:** Use regex or markdown parser; handle edge cases (lists, code, quotes). + +4. **Loading state in modal:** Show spinner during action submission; disable buttons while inflight. + +5. **Undo granularity:** Track individual action history; offer undo per-item or per-section, not just batch. + +6. **Toast notifications:** Show action result (success/error) in toast instead of relying on onResolved() refresh. + +### Low Priority +1. **Bulk actions UI:** Multi-select + batch nest/merge with preview. + +2. **Action audit log:** Log with timestamps, reversible operations, user attribution. + +3. **Tag tree integration:** Link rootless tags to tree panel with filter/highlight. + +4. **Keyboard shortcuts:** Arrow keys to navigate pairs, Enter to apply action, etc. + +5. **Export per-section:** Download individual review section as CSV/JSON for offline processing. + +--- + +## Code Locations Summary + +| Issue | File | Line(s) | Fix | +|-------|------|---------|-----| +| Check name bug | HealthReviewModal.tsx | 467 | `check: 'content_overlap'` | +| Unsafe cast | HealthReviewModal.tsx | 330 | Validate with Zod | +| Unsafe cast | HealthReviewModal.tsx | 351 | Validate with Zod | +| No empty state | HealthReviewModal.tsx | 246–254 | Add message when length=0 | +| Brittle title extract | HealthReviewModal.tsx | 223 | Use markdown parser | +| No contradiction pairs | HealthReviewModal.tsx | 318–337 | Implement pair list render | +| No quality drill-down | HealthReviewModal.tsx | 341–373 | Fetch titles, add links | +| No tag drill-down | HealthReviewModal.tsx | 377–404 | List tags, add actions | +| No tests | (new file) | — | Create `__tests__/HealthReviewModal.test.tsx` | + +--- + +## Conclusion + +**Severity:** High — one critical bug prevents merge actions from working; four tabs lack drill-down/action capability; zero test coverage. + +**Effort to Fix:** +- Critical bug: 1 line +- Tests: 1–2 days (moderate complexity, async/modal/data flow) +- Drill-down features: 2–3 days per tab (fetching, rendering, validation) +- Total: 1 week to make production-ready + +**Risk:** Currently deployed health review modal is partially non-functional (merge fails). Recommend fix and test before release. diff --git a/src/components/dashboard/widgets/HealthConfirmModal.tsx b/src/components/dashboard/widgets/HealthConfirmModal.tsx new file mode 100644 index 00000000..0624005a --- /dev/null +++ b/src/components/dashboard/widgets/HealthConfirmModal.tsx @@ -0,0 +1,77 @@ +import { createPortal } from 'react-dom'; +import { X, Play } from 'lucide-react'; +import { useEffect } from 'react'; + +export interface PendingFix { + label: string; + check: string; +} + +interface Props { + pending: PendingFix[]; + currentScore: number; + onConfirm: () => void; + onCancel: () => void; +} + +export function HealthConfirmModal({ pending, currentScore, onConfirm, onCancel }: Props) { + useEffect(() => { + const handler = (e: KeyboardEvent) => { + if (e.key === 'Escape') onCancel(); + if (e.key === 'Enter') onConfirm(); + }; + document.addEventListener('keydown', handler); + return () => document.removeEventListener('keydown', handler); + }, [onCancel, onConfirm]); + + return createPortal( +
    { if (e.target === e.currentTarget) onCancel(); }} + > +
    + {/* Header */} +
    +
    +

    Apply automatic fixes?

    +

    Current score: {currentScore}/100

    +
    + +
    + + {/* Fix list */} +
    +

    The following fixes will run:

    +
      + {pending.map((fix, i) => ( +
    • + • + {fix.label} +
      • + ))} +
    +
    + + {/* Footer */} +
    + + +
    +
    +
    , + document.body, + ); +} diff --git a/src/components/dashboard/widgets/HealthExportModal.tsx b/src/components/dashboard/widgets/HealthExportModal.tsx new file mode 100644 index 00000000..5c4e3cb1 --- /dev/null +++ b/src/components/dashboard/widgets/HealthExportModal.tsx @@ -0,0 +1,152 @@ +import { createPortal } from 'react-dom'; +import { X, Download } from 'lucide-react'; +import { useEffect } from 'react'; + +// Minimal types needed for export +interface ExportHealthCheckResult { + status: string; + score: number; + data: Record; +} + +interface ExportHealthReport { + overall_score: number; + overall_status: string; + computed_at: string; + atom_count: number; + checks: Record; + auto_fixable: number; + requires_review: number; +} + +const CHECK_LABELS_EXPORT: Record = { + embedding_coverage: 'Embeddings', + tagging_coverage: 'Tagging', + source_uniqueness: 'Source duplicates', + orphan_tags: 'Orphan tags', + semantic_graph_freshness: 'Semantic graph freshness', + wiki_coverage: 'Wiki coverage', + content_quality: 'Content quality', + tag_health: 'Tag health', + content_overlap: 'Content overlap', + contradiction_detection: 'Contradiction detection', + broken_internal_links: 'Broken internal links', + boilerplate_pollution: 'Boilerplate pollution', +}; + +const CHECK_ORDER_EXPORT = [ + 'embedding_coverage', 'tagging_coverage', 'source_uniqueness', 'orphan_tags', + 'semantic_graph_freshness', 'wiki_coverage', 'content_quality', 'tag_health', + 'content_overlap', 'contradiction_detection', 'broken_internal_links', +]; + +function buildMarkdown(report: ExportHealthReport): string { + const date = new Date(report.computed_at).toLocaleString(); + let md = `# Knowledge Base Health Report\n\n`; + md += `**Overall Score:** ${report.overall_score}/100 \n`; + md += `**Status:** ${report.overall_status.replace('_', ' ')} \n`; + md += `**Generated:** ${date} \n`; + md += `**Total atoms:** ${report.atom_count} \n\n`; + md += `---\n\n`; + + for (const key of CHECK_ORDER_EXPORT) { + const check = report.checks[key]; + if (!check) continue; + const label = CHECK_LABELS_EXPORT[key] ?? key; + const statusIcon = check.score >= 90 ? '✅' : check.score >= 70 ? '⚠️' : check.score >= 50 ? '🟠' : '❌'; + md += `## ${statusIcon} ${label}\n\n`; + md += `**Score:** ${check.score}/100 \n`; + md += `**Status:** ${check.status} \n\n`; + // Include key data fields + const dataEntries = Object.entries(check.data) + .filter(([, v]) => typeof v === 'number' || typeof v === 'string') + .slice(0, 5); + if (dataEntries.length > 0) { + for (const [k, v] of dataEntries) { + md += `- **${k.replace(/_/g, ' ')}:** ${v}\n`; + } + md += '\n'; + } + } + + return md; +} + +async function downloadMarkdown(report: ExportHealthReport): Promise { + const md = buildMarkdown(report); + const filename = `health-report-${new Date(report.computed_at).toISOString().split('T')[0]}.md`; + + // Tauri desktop: use plugin-dialog + plugin-fs if available + const tauriWindow = window as typeof window & { __TAURI__?: { dialog?: unknown; fs?: unknown } }; + if (tauriWindow.__TAURI__) { + try { + const { save } = await import('@tauri-apps/plugin-dialog'); + const { writeTextFile } = await import('@tauri-apps/plugin-fs'); + const path = await save({ + defaultPath: filename, + filters: [{ name: 'Markdown', extensions: ['md'] }], + }); + if (path) { + await writeTextFile(path, md); + } + return; + } catch { + // Fall through to web download if Tauri plugins aren't available + } + } + + // Web: data: URI download + const blob = new Blob([md], { type: 'text/markdown;charset=utf-8' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); +} + +interface Props { + report: ExportHealthReport; + onClose: () => void; +} + +export function HealthExportModal({ report, onClose }: Props) { + const md = buildMarkdown(report); + + useEffect(() => { + const handler = (e: KeyboardEvent) => { if (e.key === 'Escape') onClose(); }; + document.addEventListener('keydown', handler); + return () => document.removeEventListener('keydown', handler); + }, [onClose]); + + return createPortal( +
    { if (e.target === e.currentTarget) onClose(); }} + > +
    +
    +

    Export Health Report

    +
    + + +
    +
    +
    + 
    {md}
    +
    +
    +
    , + document.body, + ); +} diff --git a/src/components/dashboard/widgets/HealthHelpOverlay.tsx b/src/components/dashboard/widgets/HealthHelpOverlay.tsx new file mode 100644 index 00000000..7e7e0f1a --- /dev/null +++ b/src/components/dashboard/widgets/HealthHelpOverlay.tsx @@ -0,0 +1,51 @@ +import { createPortal } from 'react-dom'; +import { X } from 'lucide-react'; +import { useEffect } from 'react'; + +const SHORTCUTS = [ + { key: 'r', desc: 'Refresh all checks' }, + { key: 'f', desc: 'Open fix confirmation' }, + { key: 'e', desc: 'Export to markdown' }, + { key: '1 – 9', desc: 'Expand / collapse Nth check in list' }, + { key: '?', desc: 'Toggle this help overlay' }, + { key: 'Esc', desc: 'Close modal / overlay' }, +]; + +interface Props { + onClose: () => void; +} + +export function HealthHelpOverlay({ onClose }: Props) { + useEffect(() => { + const handler = (e: KeyboardEvent) => { + if (e.key === 'Escape' || e.key === '?') onClose(); + }; + document.addEventListener('keydown', handler); + return () => document.removeEventListener('keydown', handler); + }, [onClose]); + + return createPortal( +
    { if (e.target === e.currentTarget) onClose(); }} + > +
    +
    +

    Keyboard shortcuts

    + +
    +
    + {SHORTCUTS.map(({ key, desc }) => ( +
    + {key} + {desc} +
    + ))} +
    +
    +
    , + document.body, + ); +} diff --git a/src/components/dashboard/widgets/HealthReviewModal.tsx b/src/components/dashboard/widgets/HealthReviewModal.tsx index d112cd9e..7000efd2 100644 --- a/src/components/dashboard/widgets/HealthReviewModal.tsx +++ b/src/components/dashboard/widgets/HealthReviewModal.tsx @@ -1,7 +1,7 @@ import { useState, useEffect, useCallback } from 'react'; import { createPortal } from 'react-dom'; import { - X, GitMerge, Link, Trash2, Loader2, CheckCircle, + X, GitMerge, Link, Loader2, CheckCircle, ChevronDown, ChevronUp, ExternalLink, RefreshCw, } from 'lucide-react'; import { getTransport } from '../../../lib/transport'; @@ -23,9 +23,39 @@ interface AtomDetail { source_url?: string; } -type PairAction = 'merge_with_llm' | 'keep_both' | 'delete_older'; +type PairAction = 'merge_with_llm' | 'keep_both'; type PairStatus = 'idle' | 'loading' | 'done' | 'error'; +// Atom preview (content_quality + boilerplate) +interface AtomPreview { + id: string; + title: string; + created_at?: string; +} + +// Boilerplate atom entry +interface BoilerplateEntry { + id: string; + title: string; + clone_count: number; +} + +// Contradiction pair +interface ContradictionPair { + pair_id: string; + atom_a: { id: string; title: string; source?: string }; + atom_b: { id: string; title: string; source?: string }; + similarity: number; + shared_tag_count: number; +} + +// Rootless tag +interface RootlessTag { + id: string; + name: string; + atom_count: number; +} + // ==================== Helpers ==================== function sourceLabel(source?: string): string { @@ -89,7 +119,6 @@ function PairRow({ const labels: Record = { merge_with_llm: 'Merged — LLM synthesised both atoms into one', keep_both: 'Kept both — no changes made', - delete_older: 'Older atom deleted', }; return (
    @@ -167,15 +196,6 @@ function PairRow({ disabled={status === 'loading'} onClick={() => apply('keep_both')} /> - } - label="Delete older" - title="Delete the older atom" - loading={status === 'loading' && appliedAction === 'delete_older'} - disabled={status === 'loading'} - variant="danger" - onClick={() => apply('delete_older')} - />
    @@ -210,55 +230,21 @@ function ActionBtn({ // ==================== Boilerplate section ==================== -interface BoilerplateAtom { - id: string; - title: string; - source_url: string | null; - reembedStatus: 'idle' | 'loading' | 'done' | 'error'; -} - -function BoilerplateSection({ atomIds }: { atomIds: string[] }) { - const [atoms, setAtoms] = useState([]); - const [loadingAtoms, setLoadingAtoms] = useState(true); - - useEffect(() => { - let cancelled = false; - const fetchAll = async () => { - setLoadingAtoms(true); - const results = await Promise.allSettled( - atomIds.map(id => getTransport().invoke<{ id: string; content: string; source_url?: string }>('get_atom', { id })) - ); - if (cancelled) return; - setAtoms(results.map((r, i) => { - if (r.status === 'fulfilled') { - const first_line = r.value.content.split('\n').find(l => l.trim()) ?? atomIds[i]; - const title = first_line.replace(/^#+\s*/, '').trim().slice(0, 80); - return { id: atomIds[i], title, source_url: r.value.source_url ?? null, reembedStatus: 'idle' }; - } - return { id: atomIds[i], title: atomIds[i], source_url: null, reembedStatus: 'idle' }; - })); - setLoadingAtoms(false); - }; - fetchAll(); - return () => { cancelled = true; }; - }, [atomIds]); +function BoilerplateSection({ atoms }: { atoms: BoilerplateEntry[] }) { + const [reembedStatus, setReembedStatus] = useState>({}); const reembed = async (atomId: string) => { - setAtoms(prev => prev.map(a => a.id === atomId ? { ...a, reembedStatus: 'loading' } : a)); + setReembedStatus(prev => ({ ...prev, [atomId]: 'loading' })); try { - await getTransport().invoke('retry_embedding', { atomId: atomId }); - setAtoms(prev => prev.map(a => a.id === atomId ? { ...a, reembedStatus: 'done' } : a)); + await getTransport().invoke('retry_embedding', { atomId }); + setReembedStatus(prev => ({ ...prev, [atomId]: 'done' })); } catch { - setAtoms(prev => prev.map(a => a.id === atomId ? { ...a, reembedStatus: 'error' } : a)); + setReembedStatus(prev => ({ ...prev, [atomId]: 'error' })); } }; - if (loadingAtoms) { - return ( -
    - -
    - ); + if (atoms.length === 0) { + return

    No boilerplate pollution detected — all clear

    ; } return ( @@ -266,53 +252,210 @@ function BoilerplateSection({ atomIds }: { atomIds: string[] }) {

    Embedding quality issue

    - These {atomIds.length} atoms share identical boilerplate sections that dominate their - embeddings — semantic search cannot reliably distinguish them from each other. - Edit each atom to remove or uniquify the boilerplate sections, then re-embed. + These {atoms.length} atom{atoms.length !== 1 ? 's' : ''} share identical boilerplate sections + that dominate their embeddings — semantic search cannot reliably distinguish them from + each other. Edit each atom to remove or uniquify the boilerplate sections, then re-embed.

    +
    + {atoms + .slice() + .sort((a, b) => b.clone_count - a.clone_count) + .map(atom => { + const status = reembedStatus[atom.id] ?? 'idle'; + return ( +
    +
    +

    + {atom.title || Untitled atom} +

    +

    + {atom.clone_count} near-identical edge{atom.clone_count !== 1 ? 's' : ''} +

    +
    +
    + {status === 'done' ? ( + + Queued + + ) : status === 'error' ? ( + Failed + ) : ( + + )} +
    +
    + ); + })} +
    +
    + ); +} + +// ==================== Contradiction section ==================== + +function ContradictionRow({ pair }: { pair: ContradictionPair }) { + const [expanded, setExpanded] = useState(false); + const [contents, setContents] = useState<[string, string] | null>(null); + const [loadingContent, setLoadingContent] = useState(false); + + const toggleExpand = async () => { + if (!expanded && !contents) { + setLoadingContent(true); + try { + const [a, b] = await Promise.all([ + getTransport().invoke<{ content: string }>('get_atom', { id: pair.atom_a.id }), + getTransport().invoke<{ content: string }>('get_atom', { id: pair.atom_b.id }), + ]); + setContents([a.content, b.content]); + } catch { + setContents(['(Failed to load)', '(Failed to load)']); + } finally { + setLoadingContent(false); + } + } + setExpanded(v => !v); + }; + + const simPct = Math.round(pair.similarity * 100); + const simColor = simPct >= 88 ? 'text-orange-400' : 'text-yellow-400'; + + return ( +
    +
    +
    + {simPct}% similarity +
    + {pair.shared_tag_count > 0 && ( + {pair.shared_tag_count} shared tag{pair.shared_tag_count !== 1 ? 's' : ''} + )} + +
    +
    + +
    + {[pair.atom_a, pair.atom_b].map((atom, i) => ( +
    +

    {atom.title}

    + {atom.source && ( +

    + {(() => { try { return new URL(atom.source).hostname; } catch { return atom.source; } })()} +

    + )} +
    + ))} +
    + + {expanded && contents && ( +
    + {[pair.atom_a, pair.atom_b].map((atom, i) => ( +
    +

    {atom.title}

    + 
    +                  {contents[i as 0 | 1]}
+
    +
    + ))} +
    + )} +
    +
    + ); +} + +function ContradictionSection({ data }: { data: Record }) { + const pairs = (data.pairs as ContradictionPair[] | undefined) ?? []; + const count = (data.potential_contradictions as number) ?? 0; + if (pairs.length === 0) { + return ( +

    No contradiction candidates — all clear

    + ); + } + + return ( +
    +
    +

    Contradiction candidates

    +

    + {count} atom pair{count !== 1 ? 's' : ''} cover the same topic but may contain + conflicting information (similarity 80–92%). Compare their content and merge or + update them to align. Use Compare to view + both atoms side-by-side. +

    +
    - {atoms.map(atom => ( + {pairs.map(pair => ( + + ))} +
    +
    + ); +} + +// ==================== Content quality (no-source) section ==================== + +function ContentQualitySection({ data }: { data: Record }) { + const issues = data.issues as Record; + }> | undefined; + + const noSourceItems = (issues?.no_source?.atoms ?? []) as Array<{ id: string; title: string; created_at?: string }>; + const noSourceCount = issues?.no_source?.count ?? noSourceItems.length; + + if (noSourceCount === 0) { + return

    No unsourced atoms — all clear

    ; + } + + return ( +
    +
    +

    + {noSourceCount} atom{noSourceCount !== 1 ? 's' : ''} missing a source URL +

    +

    + These atoms have no source_url{' '} + and no URL or{' '} + Source: line in their content. + Open each atom in the editor and add a source URL to resolve. +

    +
    +
    + {noSourceItems.map(atom => (
    -

    {atom.title}

    - {atom.source_url && ( -

    {sourceLabel(atom.source_url)}

    - )} -
    -
    - {atom.source_url && ( - - - Source - - )} - {atom.reembedStatus === 'done' ? ( - - Queued - - ) : ( - +

    + {atom.title || Untitled atom} +

    + {atom.created_at && ( +

    + Created {new Date(atom.created_at).toLocaleDateString()} +

    )}
    @@ -322,7 +465,67 @@ function BoilerplateSection({ atomIds }: { atomIds: string[] }) { ); } -// ==================== Modal ==================== +// ==================== Tag health (rootless) section ==================== + +function TagHealthSection({ data }: { data: Record }) { + const rootlessList = (data.rootless_tag_list as RootlessTag[] | undefined) ?? []; + const rootlessCount = (data.rootless_tags as number) ?? rootlessList.length; + const similarCount = (data.similar_name_pairs as number) ?? 0; + + return ( +
    + {rootlessList.length > 0 && ( +
    +
    +

    + {rootlessCount} root-level tag{rootlessCount !== 1 ? 's' : ''} with no parent +

    +

    + These tags sit at the top level. Consider nesting them under a relevant + category to keep the tag tree navigable. +

    +
    +
    + {rootlessList + .slice() + .sort((a, b) => b.atom_count - a.atom_count) + .map(tag => ( +
    +
    +

    {tag.name}

    +

    + {tag.atom_count} atom{tag.atom_count !== 1 ? 's' : ''} +

    +
    +
    + ))} +
    +
    + )} + + {similarCount > 0 && ( +
    +

    + {similarCount} similar-name pair{similarCount !== 1 ? 's' : ''} +

    +

    + Tags with near-identical names (e.g. "React" and "ReactJS") may be duplicates. + Review and merge in the tag tree if needed. +

    +
    + )} + + {rootlessList.length === 0 && similarCount === 0 && ( +

    Tag structure is healthy — all clear

    + )} +
    + ); +} + +// ==================== Main modal ==================== interface Props { report: { @@ -330,25 +533,37 @@ interface Props { data: Record; }>; }; - checkName?: string; // If provided, pre-select this tab on open + checkName?: string; onClose: () => void; onResolved: () => void; } export function HealthReviewModal({ report, checkName, onClose, onResolved }: Props) { - // Compute once — stable references for the lifetime of this modal mount const overlapPairs: OverlapPair[] = (report.checks['content_overlap']?.data?.pairs as OverlapPair[]) ?? []; - const boilerplateIds: string[] = - (report.checks['boilerplate_pollution']?.data?.affected_atoms as string[]) ?? []; + const boilerplateAtoms: BoilerplateEntry[] = + (report.checks['boilerplate_pollution']?.data?.affected_atoms as BoilerplateEntry[] | undefined) ?? []; + const contradictionData: Record | null = + (report.checks['contradiction_detection']?.data ?? null) as Record | null; + const contradictionCount = (contradictionData?.potential_contradictions as number ?? 0); + const contentQualityData: Record | null = + (report.checks['content_quality']?.data ?? null) as Record | null; + const noSourceCount = (() => { + const issues = contentQualityData?.issues as Record | undefined; + return issues?.no_source?.count ?? 0; + })(); + const tagHealthData: Record | null = + (report.checks['tag_health']?.data ?? null) as Record | null; + const rootlessCount = (tagHealthData?.rootless_tags as number) ?? 0; - // Build tab list from available data const tabs = [ - ...(overlapPairs.length > 0 ? [{ key: 'content_overlap', label: 'Content overlap', count: overlapPairs.length }] : []), - ...(boilerplateIds.length > 0 ? [{ key: 'boilerplate', label: 'Boilerplate', count: boilerplateIds.length }] : []), + ...(overlapPairs.length > 0 ? [{ key: 'content_overlap', label: 'Content overlap', count: overlapPairs.length }] : []), + ...(boilerplateAtoms.length > 0 ? [{ key: 'boilerplate', label: 'Boilerplate', count: boilerplateAtoms.length }] : []), + ...(contradictionCount > 0 ? [{ key: 'contradiction_detection', label: 'Contradictions', count: contradictionCount }] : []), + ...(noSourceCount > 0 ? [{ key: 'content_quality', label: 'No source', count: noSourceCount }] : []), + ...(rootlessCount > 0 ? [{ key: 'tag_health', label: 'Tag structure', count: rootlessCount }] : []), ]; - // selectedTab = user choice; falls back to first available tab const [selectedTab, setSelectedTab] = useState(checkName ?? null); const activeTab = tabs.find(t => t.key === selectedTab)?.key ?? tabs[0]?.key ?? null; @@ -444,7 +659,19 @@ export function HealthReviewModal({ report, checkName, onClose, onResolved }: Pr )} {activeTab === 'boilerplate' && ( - + + )} + + {activeTab === 'contradiction_detection' && contradictionData && ( + + )} + + {activeTab === 'content_quality' && contentQualityData && ( + + )} + + {activeTab === 'tag_health' && tagHealthData && ( + )}
    diff --git a/src/components/dashboard/widgets/HealthWidget.tsx b/src/components/dashboard/widgets/HealthWidget.tsx index 9e2173e6..f936f95f 100644 --- a/src/components/dashboard/widgets/HealthWidget.tsx +++ b/src/components/dashboard/widgets/HealthWidget.tsx @@ -1,11 +1,15 @@ -import { useEffect, useState, useCallback } from 'react'; +import { useEffect, useState, useCallback, useRef } from 'react'; import { getTransport } from '../../../lib/transport'; import { - RefreshCw, CheckCircle, AlertTriangle, XCircle, Play, + RefreshCw, CheckCircle, AlertTriangle, XCircle, Play, Download, HelpCircle, } from 'lucide-react'; import { HealthReviewModal } from './HealthReviewModal'; import { HealthCheckRow, getTrend } from './HealthCheckRow'; import type { HealthCheckResult } from './HealthCheckRow'; +import { HealthConfirmModal } from './HealthConfirmModal'; +import type { PendingFix } from './HealthConfirmModal'; +import { HealthExportModal } from './HealthExportModal'; +import { HealthHelpOverlay } from './HealthHelpOverlay'; // ==================== Types ==================== @@ -163,6 +167,7 @@ const CHECK_ORDER = [ 'content_overlap', 'contradiction_detection', 'broken_internal_links', + 'boilerplate_pollution', ]; // ==================== Sub-components ==================== @@ -298,6 +303,11 @@ export function HealthPanel() { const [lastFix, setLastFix] = useState(null); const [error, setError] = useState(null); const [showPending, setShowPending] = useState(false); + const [showConfirm, setShowConfirm] = useState(false); + const [showExport, setShowExport] = useState(false); + const [showHelp, setShowHelp] = useState(false); + const [undoToast, setUndoToast] = useState<{ fixIds: string[]; label: string } | null>(null); + const undoTimerRef = useRef | null>(null); // Per-row state const [expandedChecks, setExpandedChecks] = useState>(new Set()); @@ -356,9 +366,14 @@ export function HealthPanel() { } }, []); - const runFix = async () => { + const runFix = () => setShowConfirm(true); + + const applyFix = async () => { + setShowConfirm(false); setFixing(true); setShowPending(false); + if (undoTimerRef.current) clearTimeout(undoTimerRef.current); + setUndoToast(null); try { const checksToFix = report ? CHECK_ORDER.filter(k => { @@ -372,6 +387,12 @@ export function HealthPanel() { checks: checksToFix, }); setLastFix(resp); + if (resp.actions_taken.length > 0) { + const fixIds = resp.actions_taken.map(a => a.id).filter(Boolean); + const label = `Fixed ${resp.actions_taken.reduce((n, a) => n + a.count, 0)} items. Score → ${resp.new_score}/100`; + setUndoToast({ fixIds, label }); + undoTimerRef.current = setTimeout(() => setUndoToast(null), 10_000); + } await fetchHealth(); } catch (err) { setError(err instanceof Error ? err.message : 'Fix failed'); @@ -380,6 +401,53 @@ export function HealthPanel() { } }; + const undoLastFix = async () => { + if (!undoToast) return; + if (undoTimerRef.current) clearTimeout(undoTimerRef.current); + setUndoToast(null); + try { + for (const fixId of [...undoToast.fixIds].reverse()) { + await getTransport().invoke('undo_health_fix', { fixId }); + } + await fetchHealth(); + } catch (err) { + setError(err instanceof Error ? err.message : 'Undo failed'); + } + }; + + // Compute these before early returns so keyboard handler can reference them + const issueChecks = report ? getVisibleChecks(report, filter) : []; + const pending: PendingFix[] = report ? pendingActions(report, excludedFromFix) : []; + const review = report ? reviewItems(report) : []; + + // Keyboard shortcuts + useEffect(() => { + const handler = (e: KeyboardEvent) => { + const tag = (e.target as HTMLElement).tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT') return; + if (showConfirm || showExport || showHelp || showReviewModal) return; + if (e.key === 'r') { + e.preventDefault(); + fetchHealth(); + } else if (e.key === 'f' && report && pending.length > 0) { + e.preventDefault(); + setShowConfirm(true); + } else if (e.key === 'e' && report) { + e.preventDefault(); + setShowExport(true); + } else if (e.key === '?') { + e.preventDefault(); + setShowHelp(v => !v); + } else if (e.key >= '1' && e.key <= '9' && issueChecks.length > 0) { + const idx = parseInt(e.key, 10) - 1; + const checkName = issueChecks[idx]; + if (checkName) toggleExpandCheck(checkName); + } + }; + document.addEventListener('keydown', handler); + return () => document.removeEventListener('keydown', handler); + }, [fetchHealth, report, pending, showConfirm, showExport, showHelp, showReviewModal, issueChecks, toggleExpandCheck]); + if (loading) { return (
    @@ -400,9 +468,7 @@ export function HealthPanel() { } const statusColor = STATUS_COLORS[report.overall_status] ?? 'text-gray-400'; - const issueChecks = getVisibleChecks(report, filter); - const pending = pendingActions(report, excludedFromFix); - const review = reviewItems(report); + return (
    @@ -419,6 +485,22 @@ export function HealthPanel() { > + +
    @@ -568,20 +650,55 @@ export function HealthPanel() {
    )} - {/* Last fix result */} + {/* Last fix result — score summary only */} {lastFix && lastFix.actions_taken.length > 0 && ( -
    -

    - Last run → score {lastFix.new_score}/100 -

    - {lastFix.actions_taken.map((a, i) => ( -

    - ✓ {FIX_ACTION_LABELS[a.action] ?? a.action.replace(/_/g, ' ')} ({a.count}) -

    - ))} +
    +

    Last run → score {lastFix.new_score}/100

    )} + {/* Undo toast */} + {undoToast && ( +
    + {undoToast.label} + + +
    + )} + + {/* Modals */} + {showConfirm && report && ( + setShowConfirm(false)} + /> + )} + {showExport && report && ( + setShowExport(false)} + /> + )} + {showHelp && ( + setShowHelp(false)} /> + )} + {/* Review modal */} {showReviewModal && report && ( ); -} +} \ No newline at end of file diff --git a/src/components/dashboard/widgets/__tests__/HealthReviewModal.test.tsx b/src/components/dashboard/widgets/__tests__/HealthReviewModal.test.tsx new file mode 100644 index 00000000..6f4d594c --- /dev/null +++ b/src/components/dashboard/widgets/__tests__/HealthReviewModal.test.tsx @@ -0,0 +1,283 @@ +/** + * HealthReviewModal tests + * + * NOTE: These tests require @testing-library/react and @testing-library/user-event. + * Install with: + * npm install -D @testing-library/react @testing-library/user-event @testing-library/jest-dom + * Then add `setupFiles: ['@testing-library/jest-dom/vitest']` to vitest.config.ts. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { HealthReviewModal } from '../HealthReviewModal'; + +// Mock transport +vi.mock('../../../../lib/transport', () => ({ + getTransport: () => ({ + invoke: vi.fn().mockResolvedValue({ content: '# Mock atom content\n\nSome text here' }), + }), +})); + +// Minimal report shape +const makeReport = (overrides: Record = {}) => ({ + checks: { + content_overlap: { + data: { + pairs: [], + cross_source_overlaps: 0, + exact_duplicates: 0, + template_clones: 0, + count: 0, + }, + }, + boilerplate_pollution: { + data: { + count: 0, + affected_atoms: [], + description: '', + }, + }, + contradiction_detection: { + data: { + pairs_checked: 0, + potential_contradictions: 0, + pairs: [], + }, + }, + content_quality: { + data: { + issues: { + no_source: { count: 0, atoms: [] }, + }, + }, + }, + tag_health: { + data: { + rootless_tags: 0, + similar_name_pairs: 0, + rootless_tag_list: [], + }, + }, + ...overrides, + }, +}); + +describe('HealthReviewModal', () => { + const onClose = vi.fn(); + const onResolved = vi.fn(); + + beforeEach(() => { + vi.clearAllMocks(); + document.body.innerHTML = ''; + }); + + it('shows nothing-to-review when all checks empty', () => { + render( + + ); + expect(screen.getByText(/nothing to review/i)).toBeTruthy(); + }); + + it('shows content overlap tab when pairs exist', () => { + const report = makeReport({ + content_overlap: { + data: { + pairs: [ + { + pair_id: 'p1', + atom_a: { id: 'a1', title: 'Article Alpha', source: 'https://site1.com/a' }, + atom_b: { id: 'b1', title: 'Article Beta', source: 'https://site2.com/b' }, + similarity: 0.72, + shared_tag_count: 3, + available_actions: ['merge_with_llm', 'keep_both'], + }, + ], + cross_source_overlaps: 1, + count: 1, + }, + }, + }); + render( + + ); + expect(screen.getByText('Content overlap')).toBeTruthy(); + expect(screen.getByText('Article Alpha')).toBeTruthy(); + expect(screen.getByText('Article Beta')).toBeTruthy(); + expect(screen.getByText('72% overlap')).toBeTruthy(); + }); + + it('shows boilerplate tab with titles and clone counts', () => { + const report = makeReport({ + boilerplate_pollution: { + data: { + count: 2, + affected_atoms: [ + { id: 'bp1', title: 'Template Article A', clone_count: 5 }, + { id: 'bp2', title: 'Template Article B', clone_count: 2 }, + ], + description: 'test', + }, + }, + }); + render( + + ); + expect(screen.getByText('Template Article A')).toBeTruthy(); + expect(screen.getByText(/5 near-identical edge/)).toBeTruthy(); + expect(screen.getByText('Template Article B')).toBeTruthy(); + expect(screen.getByText(/2 near-identical edge/)).toBeTruthy(); + }); + + it('shows contradiction tab with pair titles and similarity', () => { + const report = makeReport({ + contradiction_detection: { + data: { + pairs_checked: 50, + potential_contradictions: 1, + pairs: [ + { + pair_id: 'cp1', + atom_a: { id: 'ca1', title: 'Topic X Version 1', source: 'https://s1.com' }, + atom_b: { id: 'cb1', title: 'Topic X Version 2', source: 'https://s2.com' }, + similarity: 0.85, + shared_tag_count: 2, + }, + ], + }, + }, + }); + render( + + ); + expect(screen.getByText('Contradictions')).toBeTruthy(); + expect(screen.getByText('Topic X Version 1')).toBeTruthy(); + expect(screen.getByText('Topic X Version 2')).toBeTruthy(); + expect(screen.getByText(/85% similarity/)).toBeTruthy(); + }); + + it('shows content quality tab with atom titles', () => { + const report = makeReport({ + content_quality: { + data: { + issues: { + no_source: { + count: 2, + atoms: [ + { id: 'q1', title: 'Note Without Source', created_at: '2026-01-15T10:00:00Z' }, + { id: 'q2', title: 'Another Unsourced Note', created_at: '2026-02-01T10:00:00Z' }, + ], + }, + }, + }, + }, + }); + render( + + ); + expect(screen.getByText('No source')).toBeTruthy(); + expect(screen.getByText('Note Without Source')).toBeTruthy(); + expect(screen.getByText('Another Unsourced Note')).toBeTruthy(); + expect(screen.getByText(/1\/15\/2026|Jan 15|15 Jan/)).toBeTruthy(); + }); + + it('shows tag health tab with rootless tag names', () => { + const report = makeReport({ + tag_health: { + data: { + rootless_tags: 2, + similar_name_pairs: 0, + rootless_tag_list: [ + { id: 'tg1', name: 'Orphaned Category', atom_count: 7 }, + { id: 'tg2', name: 'Floating Topic', atom_count: 2 }, + ], + }, + }, + }); + render( + + ); + expect(screen.getByText('Tag structure')).toBeTruthy(); + expect(screen.getByText('Orphaned Category')).toBeTruthy(); + expect(screen.getByText('Floating Topic')).toBeTruthy(); + expect(screen.getByText(/7 atom/)).toBeTruthy(); + expect(screen.getByText(/2 atom/)).toBeTruthy(); + }); + + it('pre-selects tab from checkName prop', () => { + const report = makeReport({ + content_overlap: { + data: { + pairs: [{ + pair_id: 'p1', + atom_a: { id: 'a1', title: 'Alpha', source: null }, + atom_b: { id: 'b1', title: 'Beta', source: null }, + similarity: 0.70, + shared_tag_count: 2, + available_actions: ['merge_with_llm', 'keep_both'], + }], + cross_source_overlaps: 1, + count: 1, + }, + }, + boilerplate_pollution: { + data: { + count: 1, + affected_atoms: [{ id: 'bp1', title: 'Boilerplate Article', clone_count: 3 }], + description: '', + }, + }, + }); + render( + + ); + expect(screen.getByText('Boilerplate Article')).toBeTruthy(); + }); + + it('calls onClose when X button clicked', async () => { + render( + + ); + const buttons = screen.getAllByRole('button'); + if (buttons.length > 0) await userEvent.click(buttons[0]); + // verify no crash; onClose called depends on button order + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 9325e7ca..34cad748 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,6 +21,7 @@ "noFallthroughCasesInSwitch": true }, "include": ["src"], + "exclude": ["src/**/__tests__/**", "src/**/*.test.ts", "src/**/*.test.tsx"] "references": [{ "path": "./tsconfig.node.json" }] } From 79f12e399fb2ff41c57479ae6ffcfacd225e5300 Mon Sep 17 00:00:00 2001 From: bk-ty Date: Fri, 1 May 2026 13:18:40 -0500 Subject: [PATCH 03/51] feat(boilerplate): add boilerplate-aware embedding filter Strips shared boilerplate chunks from semantic search (vec_chunks) while preserving them in atom_chunks for FTS and display. - Add crates/atomic-core/src/boilerplate.rs with normalize_for_dedup, content_hash, and boilerplate_indices (with all-boilerplate fallback) - V17 migration: adds content_hash TEXT column + index on atom_chunks - save_chunks_for_atom: stores content_hash, skips vec_chunks for empty embeddings - SqliteStorage: count_chunk_hash_occurrences_impl, delete_vec_chunks_by_ids_impl, backfill_content_hashes_impl - StorageBackend: dispatch wrappers for all three new methods - process_embedding_only_inner: partition chunks, embed only non-boilerplate, save all to atom_chunks (boilerplate with empty vec) - process_existing_chunk_reembedding_batch_inner: detect and remove boilerplate vec_chunks entries in the re-embed path - Tests: 8 unit tests in boilerplate.rs + 1 integration test in health/tests.rs cargo check -p atomic-core: clean (warnings only) cargo test -p atomic-core -- boilerplate: 13 passed cargo test -p atomic-core -- health: 32 passed --- crates/atomic-core/src/boilerplate.rs | 156 +++++ crates/atomic-core/src/db.rs | 14 +- crates/atomic-core/src/embedding.rs | 98 +++- crates/atomic-core/src/health/tests.rs | 24 +- crates/atomic-core/src/lib.rs | 1 + crates/atomic-core/src/storage/mod.rs | 46 ++ .../atomic-core/src/storage/sqlite/chunks.rs | 112 +++- .../plan.md | 535 ++++++++++++++++++ package-lock.json | 152 +++++ package.json | 2 + .../dashboard/widgets/HealthReviewModal.tsx | 23 +- .../dashboard/widgets/HealthWidget.tsx | 2 +- .../__tests__/HealthReviewModal.test.tsx | 12 +- tsconfig.json | 2 +- 14 files changed, 1141 insertions(+), 38 deletions(-) create mode 100644 crates/atomic-core/src/boilerplate.rs create mode 100644 docs/plans/2026-05-01-boilerplate-aware-embedding/plan.md diff --git a/crates/atomic-core/src/boilerplate.rs b/crates/atomic-core/src/boilerplate.rs new file mode 100644 index 00000000..5c42f30e --- /dev/null +++ b/crates/atomic-core/src/boilerplate.rs @@ -0,0 +1,156 @@ +//! Boilerplate-aware embedding filter. +//! +//! Detects chunks shared across multiple atoms and excludes them from +//! semantic search vectors (vec_chunks). The stored atom content +//! (atom_chunks.content) is never modified — only the embeddings change. + +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, HashSet}; + +/// Normalize chunk text for boilerplate fingerprinting. +/// Strips markdown heading markers, collapses whitespace, lowercases. +pub(crate) fn normalize_for_dedup(text: &str) -> String { + let stripped: String = text + .lines() + .map(|l| l.trim_start_matches('#').trim()) + .collect::>() + .join(" "); + stripped + .split_whitespace() + .collect::>() + .join(" ") + .to_lowercase() +} + +/// Compute SHA-256 hex digest of the normalized chunk text. +pub(crate) fn content_hash(text: &str) -> String { + let normalized = normalize_for_dedup(text); + let mut hasher = Sha256::new(); + hasher.update(normalized.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +/// Given a map of `hash → distinct_atom_count`, return the set of chunk +/// indices that are boilerplate (count >= min_atom_threshold). +/// +/// **Fallback:** if every chunk would be filtered, returns an empty set +/// so atoms with 100% boilerplate content still get embedded. +pub(crate) fn boilerplate_indices( + chunks: &[String], + counts: &HashMap, + min_atom_threshold: i64, +) -> HashSet { + if min_atom_threshold <= 0 { + return HashSet::new(); + } + let indices: HashSet = chunks + .iter() + .enumerate() + .filter_map(|(i, chunk)| { + let h = content_hash(chunk); + let count = counts.get(&h).copied().unwrap_or(0); + (count >= min_atom_threshold).then_some(i) + }) + .collect(); + // Fallback: never strip all chunks + if indices.len() == chunks.len() && !chunks.is_empty() { + HashSet::new() + } else { + indices + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_normalize_strips_heading_markers() { + assert_eq!(normalize_for_dedup("# My Header"), "my header"); + assert_eq!(normalize_for_dedup("## Section"), "section"); + } + + #[test] + fn test_normalize_collapses_whitespace() { + assert_eq!(normalize_for_dedup(" hello world "), "hello world"); + } + + #[test] + fn test_normalize_lowercases() { + assert_eq!(normalize_for_dedup("Hello World"), "hello world"); + } + + #[test] + fn test_content_hash_deterministic() { + let h1 = content_hash("# My Header"); + let h2 = content_hash("# My Header"); + assert_eq!(h1, h2); + assert_eq!(h1.len(), 64); // SHA-256 hex + } + + #[test] + fn test_content_hash_normalizes_heading_variants() { + // Different markdown levels with same text → same hash after normalization + let h1 = content_hash("# Terms of Service"); + let h2 = content_hash("## Terms of Service"); + assert_eq!(h1, h2); + } + + #[test] + fn test_boilerplate_indices_all_unique() { + let chunks = vec![ + "unique content a".to_string(), + "unique content b".to_string(), + ]; + let counts: HashMap = HashMap::new(); + let indices = boilerplate_indices(&chunks, &counts, 5); + assert!(indices.is_empty()); + } + + #[test] + fn test_boilerplate_indices_shared_chunks() { + let chunks = vec![ + "shared header".to_string(), + "unique body content".to_string(), + "shared footer".to_string(), + ]; + let mut counts = HashMap::new(); + counts.insert(content_hash("shared header"), 10i64); + counts.insert(content_hash("shared footer"), 8i64); + let indices = boilerplate_indices(&chunks, &counts, 5); + assert_eq!(indices, HashSet::from([0, 2])); + } + + #[test] + fn test_boilerplate_indices_fallback_all_boilerplate() { + let chunks = vec![ + "shared chunk a".to_string(), + "shared chunk b".to_string(), + ]; + let mut counts = HashMap::new(); + counts.insert(content_hash("shared chunk a"), 20i64); + counts.insert(content_hash("shared chunk b"), 15i64); + // All chunks are boilerplate → fallback: return empty set + let indices = boilerplate_indices(&chunks, &counts, 5); + assert!(indices.is_empty(), "should fall back to empty when all chunks are boilerplate"); + } + + #[test] + fn test_boilerplate_below_threshold_not_filtered() { + let chunks = vec!["shared header".to_string()]; + let mut counts = HashMap::new(); + counts.insert(content_hash("shared header"), 3i64); // below threshold of 5 + let indices = boilerplate_indices(&chunks, &counts, 5); + assert!(indices.is_empty()); + } + + #[test] + fn test_boilerplate_threshold_zero_disabled() { + let chunks = vec!["any content".to_string()]; + let mut counts = HashMap::new(); + counts.insert(content_hash("any content"), 100i64); + // threshold = 0 means disabled → nothing filtered + let indices = boilerplate_indices(&chunks, &counts, 0); + assert!(indices.is_empty()); + } +} diff --git a/crates/atomic-core/src/db.rs b/crates/atomic-core/src/db.rs index f3918398..9d7b8b6f 100644 --- a/crates/atomic-core/src/db.rs +++ b/crates/atomic-core/src/db.rs @@ -211,7 +211,7 @@ impl Database { /// 1. Add a new `if version < N` block at the end (before the virtual-table section) /// 2. End the block with `PRAGMA user_version = N;` /// 3. Bump LATEST_VERSION - const LATEST_VERSION: i32 = 16; + const LATEST_VERSION: i32 = 17; pub fn run_migrations(conn: &Connection) -> Result<(), AtomicCoreError> { Self::run_migrations_internal(conn, false) @@ -856,6 +856,18 @@ impl Database { )?; } + // --- V16 → V17: content_hash column on atom_chunks for boilerplate detection --- + if version < 17 { + conn.execute_batch( + r#" + ALTER TABLE atom_chunks ADD COLUMN content_hash TEXT; + CREATE INDEX IF NOT EXISTS idx_atom_chunks_content_hash + ON atom_chunks(content_hash); + PRAGMA user_version = 17; + "#, + )?; + } + // --- Triggers (recreated every startup to stay current) --- conn.execute_batch( "DROP TRIGGER IF EXISTS atom_tags_insert_count; diff --git a/crates/atomic-core/src/embedding.rs b/crates/atomic-core/src/embedding.rs index 71ddc019..28246bd9 100644 --- a/crates/atomic-core/src/embedding.rs +++ b/crates/atomic-core/src/embedding.rs @@ -582,16 +582,49 @@ async fn process_embedding_only_inner( return Ok(()); } - // Use adaptive batching so provider batch-size limits (e.g. DashScope's - // max 10) are handled by splitting, same as the bulk embedding path. - let pending: Vec = chunks + // ---- Boilerplate filtering ---- + // Exclude chunks shared across >= threshold distinct atoms from vec_chunks. + // They are still saved to atom_chunks (for FTS/display); only embedding is skipped. + let threshold = settings_map + .get("boilerplate_min_atom_count") + .and_then(|v| v.parse::().ok()) + .unwrap_or(5); + let boilerplate_set: std::collections::HashSet = if threshold > 0 && !chunks.is_empty() { + let hashes: Vec = chunks + .iter() + .map(|c| crate::boilerplate::content_hash(c)) + .collect(); + let counts = storage + .count_chunk_hash_occurrences_sync(&hashes) + .await + .unwrap_or_default(); + crate::boilerplate::boilerplate_indices(&chunks, &counts, threshold) + } else { + std::collections::HashSet::new() + }; + if !boilerplate_set.is_empty() { + tracing::debug!( + atom_id, + stripped = boilerplate_set.len(), + total = chunks.len(), + "Boilerplate filter: excluding shared chunks from embedding" + ); + } + + // Partition chunks: embed only non-boilerplate ones. + // Boilerplate chunks are saved to atom_chunks with empty embedding (skipped from vec_chunks). + let (embed_chunks, skip_chunks): (Vec<(usize, String)>, Vec<(usize, String)>) = chunks .into_iter() .enumerate() + .partition(|(index, _)| !boilerplate_set.contains(index)); + + let pending: Vec = embed_chunks + .iter() .map(|(index, chunk)| PendingChunk { atom_id: atom_id.to_string(), existing_chunk_id: None, - chunk_index: index, - content: chunk, + chunk_index: *index, + content: chunk.clone(), }) .collect(); @@ -607,12 +640,16 @@ async fn process_embedding_only_inner( } // Store chunks and embeddings - let chunks_with_embeddings: Vec<(String, Vec)> = embedded + // Boilerplate chunks (skip_chunks) are saved with empty vec → atom_chunks only, no vec_chunks. + let mut all_chunks_for_save: Vec<(String, Vec)> = embedded .into_iter() .map(|(chunk, emb)| (chunk.content, emb)) .collect(); + for (_, boilerplate_content) in skip_chunks { + all_chunks_for_save.push((boilerplate_content, vec![])); + } storage - .save_chunks_and_embeddings_sync(atom_id, &chunks_with_embeddings) + .save_chunks_and_embeddings_sync(atom_id, &all_chunks_for_save) .await .map_err(|e| format!("Failed to store chunks: {}", e))?; @@ -1708,6 +1745,53 @@ where chunks.sort_by_key(|chunk| chunk.chunk_index); } + + // ---- Boilerplate filtering for re-embed path ---- + { + let threshold = settings_map + .get("boilerplate_min_atom_count") + .and_then(|v| v.parse::().ok()) + .unwrap_or(5); + if threshold > 0 { + let all_hashes: Vec = { + let hash_set: std::collections::HashSet = atom_groups + .iter() + .flat_map(|(_, chunks)| chunks.iter().map(|c| crate::boilerplate::content_hash(&c.content))) + .collect(); + hash_set.into_iter().collect() + }; + let occurrence_counts = storage + .count_chunk_hash_occurrences_sync(&all_hashes) + .await + .unwrap_or_default(); + let mut boilerplate_chunk_ids: Vec = Vec::new(); + for (_, chunks) in &mut atom_groups { + let texts: Vec = chunks.iter().map(|c| c.content.clone()).collect(); + let bp_indices = crate::boilerplate::boilerplate_indices(&texts, &occurrence_counts, threshold); + if !bp_indices.is_empty() { + for idx in &bp_indices { + if let Some(chunk) = chunks.get(*idx) { + if let Some(ref id) = chunk.existing_chunk_id { + boilerplate_chunk_ids.push(id.clone()); + } + } + } + let kept: Vec = chunks + .drain(..) + .enumerate() + .filter(|(i, _)| !bp_indices.contains(i)) + .map(|(_, c)| c) + .collect(); + *chunks = kept; + } + } + if !boilerplate_chunk_ids.is_empty() { + if let Err(e) = storage.delete_vec_chunks_by_ids_sync(&boilerplate_chunk_ids).await { + tracing::warn!(error = %e, "Failed to delete boilerplate vec_chunks entries"); + } + } + } + } let mut chunk_groups: Vec)>> = Vec::new(); let mut current_group = Vec::new(); let mut current_chunk_count = 0usize; diff --git a/crates/atomic-core/src/health/tests.rs b/crates/atomic-core/src/health/tests.rs index f0b2f27a..43018068 100644 --- a/crates/atomic-core/src/health/tests.rs +++ b/crates/atomic-core/src/health/tests.rs @@ -226,7 +226,9 @@ mod tests { assert_eq!(pairs.len(), 1); assert_eq!(pairs[0]["pair_id"], "cp1"); assert_eq!(pairs[0]["atom_a"]["title"], "Article on Topic X - Version 1"); - assert_eq!(pairs[0]["similarity"], 0.85); + // f32 serializes with limited precision; compare as f64 with tolerance + let sim = pairs[0]["similarity"].as_f64().unwrap(); + assert!((sim - 0.85).abs() < 0.001, "expected ~0.85, got {sim}"); } // --- tag_health --- @@ -317,4 +319,24 @@ mod tests { // tagging = 0.0 * 0.20 + others = 1.0 * 0.80 → 80 assert_eq!(score, 80); } + + // --- boilerplate_indices integration --- + + #[test] + fn test_boilerplate_filtering_preserves_unique_chunks() { + use crate::boilerplate::{boilerplate_indices, content_hash}; + use std::collections::HashMap; + let chunks = vec![ + "# Privacy Policy\n\nAll rights reserved.".to_string(), + "This atom is about machine learning and neural networks.".to_string(), + "# Privacy Policy\n\nAll rights reserved.".to_string(), + ]; + let mut counts = HashMap::new(); + let bp_hash = content_hash("# Privacy Policy\n\nAll rights reserved."); + counts.insert(bp_hash, 20i64); + let indices = boilerplate_indices(&chunks, &counts, 5); + assert!(indices.contains(&0)); + assert!(!indices.contains(&1)); + assert!(indices.contains(&2)); + } } diff --git a/crates/atomic-core/src/lib.rs b/crates/atomic-core/src/lib.rs index 2d347359..87050527 100644 --- a/crates/atomic-core/src/lib.rs +++ b/crates/atomic-core/src/lib.rs @@ -30,6 +30,7 @@ pub mod agent; pub(crate) mod atom_links; pub mod briefing; +pub(crate) mod boilerplate; pub mod canvas_level; pub mod chat; pub mod chunking; diff --git a/crates/atomic-core/src/storage/mod.rs b/crates/atomic-core/src/storage/mod.rs index 7e76af9e..585baa8d 100644 --- a/crates/atomic-core/src/storage/mod.rs +++ b/crates/atomic-core/src/storage/mod.rs @@ -332,6 +332,52 @@ impl StorageBackend { StorageBackend::Postgres(_) => Ok(None), } } + pub(crate) async fn count_chunk_hash_occurrences_sync( + &self, + hashes: &[String], + ) -> Result, AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + let hashes = hashes.to_vec(); + tokio::task::spawn_blocking(move || s.count_chunk_hash_occurrences_impl(&hashes)) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(std::collections::HashMap::new()), + } + } + + pub(crate) async fn delete_vec_chunks_by_ids_sync( + &self, + chunk_ids: &[String], + ) -> Result<(), AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + let chunk_ids = chunk_ids.to_vec(); + tokio::task::spawn_blocking(move || s.delete_vec_chunks_by_ids_impl(&chunk_ids)) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(()), + } + } + + pub(crate) async fn backfill_content_hashes_sync(&self) -> Result { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + tokio::task::spawn_blocking(move || s.backfill_content_hashes_impl()) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(0), + } + } } // ==================== Async dispatch methods ==================== diff --git a/crates/atomic-core/src/storage/sqlite/chunks.rs b/crates/atomic-core/src/storage/sqlite/chunks.rs index 3cfb6346..b913c1c7 100644 --- a/crates/atomic-core/src/storage/sqlite/chunks.rs +++ b/crates/atomic-core/src/storage/sqlite/chunks.rs @@ -247,17 +247,24 @@ impl SqliteStorage { // Insert new chunks and embeddings for (index, (chunk_content, embedding_vec)) in chunks.iter().enumerate() { let chunk_id = Uuid::new_v4().to_string(); - let embedding_blob = embedding::f32_vec_to_blob_public(embedding_vec); + let hash = crate::boilerplate::content_hash(chunk_content); + let embedding_blob = if embedding_vec.is_empty() { + None::> + } else { + Some(embedding::f32_vec_to_blob_public(embedding_vec)) + }; conn.execute( - "INSERT INTO atom_chunks (id, atom_id, chunk_index, content, embedding) VALUES (?1, ?2, ?3, ?4, ?5)", - rusqlite::params![&chunk_id, atom_id, index as i32, chunk_content, &embedding_blob], + "INSERT INTO atom_chunks (id, atom_id, chunk_index, content, content_hash, embedding) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + rusqlite::params![&chunk_id, atom_id, index as i32, chunk_content, &hash, &embedding_blob], )?; - conn.execute( - "INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?1, ?2)", - rusqlite::params![&chunk_id, &embedding_blob], - )?; + if let Some(ref blob) = embedding_blob { + conn.execute( + "INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?1, ?2)", + rusqlite::params![&chunk_id, blob], + )?; + } } // Incrementally update FTS index @@ -1171,6 +1178,97 @@ impl SqliteStorage { tagging_failed, }) } + /// Given a list of content_hash values, return map of hash → count of distinct + /// atoms containing a chunk with that hash. Used for boilerplate detection. + pub(crate) fn count_chunk_hash_occurrences_impl( + &self, + hashes: &[String], + ) -> StorageResult> { + if hashes.is_empty() { + return Ok(std::collections::HashMap::new()); + } + let conn = self.db.read_conn()?; + let placeholders = hashes.iter().map(|_| "?").collect::>().join(","); + let sql = format!( + "SELECT content_hash, COUNT(DISTINCT atom_id) as cnt + FROM atom_chunks + WHERE content_hash IN ({placeholders}) + AND content_hash IS NOT NULL + GROUP BY content_hash" + ); + let mut stmt = conn.prepare(&sql)?; + let mut map = std::collections::HashMap::new(); + let rows = stmt.query_map( + rusqlite::params_from_iter(hashes.iter()), + |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)), + )?; + for row in rows { + let (hash, cnt) = row?; + map.insert(hash, cnt); + } + Ok(map) + } + + /// Delete vec_chunks entries for specific chunk IDs. + /// Used after boilerplate detection to remove vectors for shared chunks. + pub(crate) fn delete_vec_chunks_by_ids_impl( + &self, + chunk_ids: &[String], + ) -> StorageResult<()> { + if chunk_ids.is_empty() { + return Ok(()); + } + let mut conn = self + .db + .conn + .lock() + .map_err(|e| AtomicCoreError::Lock(e.to_string()))?; + let tx = conn + .transaction() + .map_err(|e| AtomicCoreError::DatabaseOperation(e.to_string()))?; + let placeholders = chunk_ids.iter().map(|_| "?").collect::>().join(","); + let sql = format!("DELETE FROM vec_chunks WHERE chunk_id IN ({placeholders})"); + tx.execute(&sql, rusqlite::params_from_iter(chunk_ids.iter()))?; + tx.commit().map_err(|e| AtomicCoreError::DatabaseOperation(e.to_string()))?; + Ok(()) + } + + /// Backfill content_hash for all atom_chunks rows that have content but NULL hash. + /// Safe to run multiple times (idempotent). Returns number of rows updated. + pub(crate) fn backfill_content_hashes_impl(&self) -> StorageResult { + use crate::boilerplate::content_hash; + let conn = self.db.read_conn()?; + let ids_and_contents: Vec<(String, String)> = { + let mut stmt = conn.prepare( + "SELECT id, content FROM atom_chunks WHERE content_hash IS NULL LIMIT 5000", + )?; + let x = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect::, _>>()?; + x + }; + drop(conn); + if ids_and_contents.is_empty() { + return Ok(0); + } + let mut write_conn = self + .db + .conn + .lock() + .map_err(|e| AtomicCoreError::Lock(e.to_string()))?; + let tx = write_conn + .transaction() + .map_err(|e| AtomicCoreError::DatabaseOperation(e.to_string()))?; + let count = ids_and_contents.len(); + for (id, content) in &ids_and_contents { + let hash = content_hash(content); + tx.execute( + "UPDATE atom_chunks SET content_hash = ?1 WHERE id = ?2", + rusqlite::params![hash, id], + )?; + } + tx.commit().map_err(|e| AtomicCoreError::DatabaseOperation(e.to_string()))?; + Ok(count) + } } #[async_trait] diff --git a/docs/plans/2026-05-01-boilerplate-aware-embedding/plan.md b/docs/plans/2026-05-01-boilerplate-aware-embedding/plan.md new file mode 100644 index 00000000..a3b4f9d4 --- /dev/null +++ b/docs/plans/2026-05-01-boilerplate-aware-embedding/plan.md @@ -0,0 +1,535 @@ +# Boilerplate-Aware Embedding + +**Date:** 2026-05-01 +**Status:** Planning +**Project:** Atomic +**Request:** Strip boilerplate chunks before embedding without changing stored atom content (Option 2). Re-embed should work correctly after this change. + +--- + +## Executive Summary + +Atoms that share identical boilerplate sections (headers, footers, disclaimers) generate near-identical embedding vectors because those tokens dominate the vector space. The fix: detect shared chunks at embedding time, exclude them from `vec_chunks` (the semantic search index) while keeping them in `atom_chunks` (FTS and display). Stored atom content is never modified. + +The Re-embed button in the health dashboard then becomes meaningful — it will re-run the pipeline with boilerplate filtering, producing distinct vectors for atoms whose unique content had previously been drowned out. + +--- + +## Current Architecture & Evidence + +### Embedding pipeline: single atom (`embedding.rs` L511–638) + +``` +chunk_content(content) ← chunking.rs:457 + → Vec + → PendingChunk { atom_id, chunk_index, content } +embed_chunks_batched(provider, pending) ← sends chunk.content to provider + → Vec<(PendingChunk, Vec)> +save_chunks_and_embeddings_sync(atom_id, [(content, vec)]) + → atom_chunks(id, atom_id, chunk_index, content, embedding) + → vec_chunks(chunk_id, embedding) ← semantic search index +``` + +**Injection point:** `embedding.rs:559` — after `chunk_content`, before building `pending`. + +### Re-embed path (`embedding.rs` L1630–1824) + +Used by the Re-embed button via `retry_embedding`. Loads existing `atom_chunks.content` from the DB and sends those texts to the provider again, then calls `update_chunk_embeddings_sync` which updates both `atom_chunks.embedding` and `vec_chunks.embedding`. + +**Injection point:** `embedding.rs:1679–1688` — after loading existing chunks, before building `group_chunks`. + +### Chunk storage schema (inferred from `chunks.rs` L100–175, L224) + +```sql +atom_chunks (id, atom_id, chunk_index, content, embedding) +vec_chunks (chunk_id, embedding) -- sqlite-vec virtual table, drives semantic search +``` + +`atom_chunks` is also indexed for FTS (`fts_atom_chunks`). `vec_chunks` is the semantic search source. These are currently in sync — every chunk has both a content entry and a vector entry. + +### Boilerplate detection query (currently in `health.rs` L394–408) + +```sql +SELECT source_atom_id FROM semantic_edges +WHERE similarity_score >= 0.99 +GROUP BY source_atom_id HAVING COUNT(*) >= 2 +LIMIT 50 +``` + +This detects the *symptom* (near-identical edge scores) but does nothing about the cause at embedding time. + +--- + +## Recommended Approach + +### Option 2: Strip boilerplate chunks before embedding, preserve stored content + +**Core idea:** compute a normalized fingerprint for each chunk, count how many distinct atoms share that exact chunk, and skip sending it to the embedding provider if it appears in ≥ N atoms. The chunk stays in `atom_chunks` (FTS still works) but gets no entry in `vec_chunks` (semantic search ignores it). + +**Threshold:** 5 atoms (configurable via settings key `boilerplate_min_atom_count`, default `5`). + +**Normalization:** lowercase + collapse whitespace + strip leading `#` markdown markers. This ensures `# My Header` and `## My Header` with different whitespace are treated as the same boilerplate. + +**Fast detection:** add a `content_hash TEXT` column to `atom_chunks` (SHA-256 of normalized text, stored as hex). Index it. One GROUP BY query per embedding run tells us which hashes appear in ≥ N atoms. + +--- + +## Implementation Plan + +### Phase 0: New `boilerplate.rs` module (~2h) + +**File:** `crates/atomic-core/src/boilerplate.rs` + +```rust +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, HashSet}; + +/// Normalize chunk text for boilerplate detection. +/// Lowercases, collapses whitespace, strips leading markdown heading markers. +pub fn normalize_for_dedup(text: &str) -> String { + text.lines() + .map(|l| l.trim_start_matches('#').trim()) + .collect::>() + .join(" ") + .split_whitespace() + .collect::>() + .join(" ") + .to_lowercase() +} + +/// Compute SHA-256 hex digest of normalized text. +pub fn content_hash(text: &str) -> String { + let normalized = normalize_for_dedup(text); + let mut hasher = Sha256::new(); + hasher.update(normalized.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +/// Given a map of `hash → distinct_atom_count`, return the indices of chunks +/// that are boilerplate (count >= threshold). +/// If ALL chunks would be filtered, returns an empty set (fallback: embed everything). +pub fn boilerplate_indices( + chunks: &[String], + counts: &HashMap, + min_atom_threshold: i64, +) -> HashSet { + let indices: HashSet = chunks + .iter() + .enumerate() + .filter_map(|(i, chunk)| { + let h = content_hash(chunk); + let count = counts.get(&h).copied().unwrap_or(0); + (count >= min_atom_threshold).then_some(i) + }) + .collect(); + + // Fallback: if every chunk is boilerplate, embed all of them + // (better than producing a zero-chunk atom with no vector) + if indices.len() == chunks.len() { + HashSet::new() + } else { + indices + } +} +``` + +Add `sha2` to `[dependencies]` in `crates/atomic-core/Cargo.toml` (already likely present — verify). + +Declare the module in `crates/atomic-core/src/lib.rs`: +```rust +pub(crate) mod boilerplate; +``` + +--- + +### Phase 1: Schema migration — add `content_hash` to `atom_chunks` (~1h) + +**File:** `crates/atomic-core/src/db.rs` (SQLite schema migrations) + +Find the latest migration version (currently V10 based on the `011_edges_status.sql` Postgres mirror). Add a new SQLite migration: + +```rust +// V11: add content_hash column to atom_chunks for boilerplate detection +conn.execute_batch( + "ALTER TABLE atom_chunks ADD COLUMN content_hash TEXT; + CREATE INDEX IF NOT EXISTS idx_atom_chunks_content_hash + ON atom_chunks(content_hash);", +)?; +``` + +This is a safe `ADD COLUMN` (nullable, no default required). Existing rows will have `content_hash = NULL` until re-embedded. + +--- + +### Phase 2: Write content_hash when saving chunks (~1h) + +**File:** `crates/atomic-core/src/storage/sqlite/chunks.rs`, `save_chunks_for_atom` (L224) + +Update the INSERT to compute and store the hash: + +```rust +use crate::boilerplate::content_hash; + +// In save_chunks_for_atom, when inserting each chunk: +let hash = content_hash(&content); +conn.execute( + "INSERT INTO atom_chunks (id, atom_id, chunk_index, content, content_hash, embedding) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![chunk_id, atom_id, idx, content, hash, embedding_blob], +)?; +``` + +--- + +### Phase 3: Storage helper for boilerplate count lookup (~1h) + +**File:** `crates/atomic-core/src/storage/sqlite/chunks.rs` + +Add a new sync method: + +```rust +/// Given a list of content hashes, return a map of hash → count of distinct atoms +/// that contain a chunk with that hash. Used for boilerplate detection at embed time. +pub(crate) fn count_chunk_hash_occurrences_sync( + &self, + hashes: &[String], +) -> StorageResult> { + if hashes.is_empty() { + return Ok(HashMap::new()); + } + let conn = self.db.read_conn()?; + let placeholders = hashes.iter().map(|_| "?").collect::>().join(","); + let sql = format!( + "SELECT content_hash, COUNT(DISTINCT atom_id) as cnt + FROM atom_chunks + WHERE content_hash IN ({}) + AND content_hash IS NOT NULL + GROUP BY content_hash", + placeholders + ); + let mut stmt = conn.prepare(&sql)?; + let mut map = HashMap::new(); + let rows = stmt.query_map( + rusqlite::params_from_iter(hashes.iter()), + |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)), + )?; + for row in rows { + let (hash, cnt) = row?; + map.insert(hash, cnt); + } + Ok(map) +} +``` + +Wire this up through the `StorageBackend` async wrapper in `ChunkStore` trait and `StorageBackend` dispatcher as `count_chunk_hash_occurrences`. + +--- + +### Phase 4: Inject filtering into single-atom embedding (`process_embedding_only_inner`) (~1.5h) + +**File:** `crates/atomic-core/src/embedding.rs` L559–596 + +After `let chunks = chunk_content(content)`: + +```rust +// Boilerplate filtering: exclude chunks shared across >= threshold atoms +let threshold = settings_map + .get("boilerplate_min_atom_count") + .and_then(|v| v.parse::().ok()) + .unwrap_or(5); + +let hashes: Vec = chunks.iter().map(|c| boilerplate::content_hash(c)).collect(); +let occurrence_counts = storage + .count_chunk_hash_occurrences_sync(&hashes) + .await + .unwrap_or_default(); +let boilerplate_set = boilerplate::boilerplate_indices(&chunks, &occurrence_counts, threshold); + +if !boilerplate_set.is_empty() { + tracing::debug!( + atom_id, + stripped = boilerplate_set.len(), + total = chunks.len(), + "Stripping boilerplate chunks before embedding" + ); +} + +let pending: Vec = chunks + .into_iter() + .enumerate() + .filter(|(i, _)| !boilerplate_set.contains(i)) + .map(|(index, chunk)| PendingChunk { atom_id: atom_id.to_string(), existing_chunk_id: None, chunk_index: index, content: chunk }) + .collect(); +``` + +> **Note:** chunks are still saved to `atom_chunks` (FTS) after this — the filter only affects what gets embedded. The `save_chunks_and_embeddings_sync` call needs to save ALL chunks to `atom_chunks` but only boilerplate-filtered ones to `vec_chunks`. + +**Required change to `save_chunks_and_embeddings_sync` / `save_chunks_for_atom`:** + +Change the signature to accept a `boilerplate_indices: &HashSet` parameter. When inserting a chunk whose index is in `boilerplate_set`, insert into `atom_chunks` with `embedding = NULL` and skip the `vec_chunks` insert. + +Alternatively (simpler): save all chunks with embeddings as today, but after saving, delete `vec_chunks` entries for boilerplate chunks. This avoids changing the save signature. + +Recommended: the "delete after save" approach for minimal blast radius: + +```rust +// After save_chunks_and_embeddings_sync, delete vec_chunks for boilerplate chunk indices +if !boilerplate_set.is_empty() { + storage.delete_boilerplate_chunk_vectors_sync(atom_id, &boilerplate_set).await.ok(); +} +``` + +New storage method `delete_boilerplate_chunk_vectors_sync(atom_id, indices)`: +```sql +DELETE FROM vec_chunks +WHERE chunk_id IN ( + SELECT id FROM atom_chunks + WHERE atom_id = ?1 AND chunk_index IN (?,?,...) +) +``` + +--- + +### Phase 5: Inject filtering into re-embed path (`process_existing_chunk_reembedding_batch_inner`) (~1.5h) + +**File:** `crates/atomic-core/src/embedding.rs` L1678–1700 + +After loading `existing_chunks` and building `chunks_by_atom`, add boilerplate filtering per atom: + +```rust +// Bulk-fetch occurrence counts for all chunk hashes in this group +let all_hashes: Vec = chunks_by_atom + .values() + .flat_map(|chunks| chunks.iter().map(|c| boilerplate::content_hash(&c.content))) + .collect::>() + .into_iter() + .collect(); + +let occurrence_counts = storage + .count_chunk_hash_occurrences_sync(&all_hashes) + .await + .unwrap_or_default(); + +// Filter boilerplate per atom's chunk list +let mut boilerplate_chunk_ids: Vec = Vec::new(); +for (atom_id, chunks) in &mut chunks_by_atom { + let texts: Vec = chunks.iter().map(|c| c.content.clone()).collect(); + let bp_indices = boilerplate::boilerplate_indices(&texts, &occurrence_counts, threshold); + if !bp_indices.is_empty() { + for i in &bp_indices { + if let Some(chunk) = chunks.get(*i) { + if let Some(ref id) = chunk.existing_chunk_id { + boilerplate_chunk_ids.push(id.clone()); + } + } + } + // Remove boilerplate chunks from the re-embed list + let mut keep = chunks.drain(..).enumerate() + .filter(|(i, _)| !bp_indices.contains(i)) + .map(|(_, c)| c) + .collect::>(); + *chunks = keep; + } +} + +// Delete vec_chunks entries for boilerplate chunk IDs +if !boilerplate_chunk_ids.is_empty() { + storage.delete_vec_chunks_by_ids_sync(&boilerplate_chunk_ids).await.ok(); +} +``` + +New storage method `delete_vec_chunks_by_ids_sync(chunk_ids: &[String])`: +```sql +DELETE FROM vec_chunks WHERE chunk_id IN (?, ?, ...) +``` + +--- + +### Phase 6: Backfill `content_hash` for existing atoms (~0.5h) + +Existing `atom_chunks` rows have `content_hash = NULL`. They need hashes so boilerplate detection works on the first re-embed run. Add a one-time backfill function: + +```rust +/// Backfill content_hash for all atom_chunks rows that have content but no hash. +/// Called once at startup (skip if all rows already have hashes). +pub(crate) fn backfill_content_hashes_sync(&self) -> StorageResult +``` + +```sql +-- Read rows needing backfill +SELECT id, content FROM atom_chunks WHERE content_hash IS NULL LIMIT 1000 +-- Update in batches of 1000 +UPDATE atom_chunks SET content_hash = ? WHERE id = ? +``` + +Do this in a background task at server startup (in `main.rs` or the health task scheduler), not blocking the hot path. + +--- + +### Phase 7: Update health dashboard Re-embed UX (~0.5h) + +**File:** `src/components/dashboard/widgets/HealthReviewModal.tsx`, `BoilerplateSection` + +- Change the button label from **"Re-embed"** to **"Re-embed (strip boilerplate)"** with a tooltip explaining what it does +- After re-embed queues, show a more informative message: `"Queued — boilerplate will be stripped from embedding on next pipeline run"` +- Remove the confusing explanatory text telling users to "edit each atom" + +--- + +## Files / Components To Change + +| File | Change | +|------|--------| +| `crates/atomic-core/Cargo.toml` | Add `sha2` dependency if not present | +| `crates/atomic-core/src/boilerplate.rs` | **New** — normalize, hash, filter logic | +| `crates/atomic-core/src/lib.rs` | Declare `pub(crate) mod boilerplate` | +| `crates/atomic-core/src/db.rs` | V11 migration: add `content_hash` column + index | +| `crates/atomic-core/src/storage/sqlite/chunks.rs` | `save_chunks_for_atom` stores hash; new `count_chunk_hash_occurrences_sync`; new `delete_vec_chunks_by_ids_sync`; new `delete_boilerplate_chunk_vectors_sync`; new `backfill_content_hashes_sync` | +| `crates/atomic-core/src/storage/traits.rs` | Add new storage trait methods | +| `crates/atomic-core/src/embedding.rs` | Filter in `process_embedding_only_inner` (L559) and `process_existing_chunk_reembedding_batch_inner` (L1679) | +| `crates/atomic-core/src/health/checks.rs` | `boilerplate_pollution` description update (minor) | +| `src/components/dashboard/widgets/HealthReviewModal.tsx` | Update Re-embed button label and success message | + +--- + +## Data Flow / Interfaces + +``` +chunk_content(content) + → Vec [all chunks, original text] + +boilerplate_indices(chunks, counts, threshold) + → HashSet [indices to skip for embedding] + +embed_chunks_batched(provider, non_boilerplate_pending) + → Vec<(PendingChunk, Vec)> [vectors for unique chunks only] + +save_chunks_and_embeddings_sync(atom_id, all_chunks_with_vecs) + → atom_chunks: all chunks (FTS intact) + → vec_chunks: all chunks initially + +delete_boilerplate_chunk_vectors_sync(atom_id, boilerplate_indices) + → vec_chunks: boilerplate chunk entries removed +``` + +--- + +## Configuration + +New settings key: `boilerplate_min_atom_count` (default: `"5"`) + +- Stored in `settings` table like all other settings +- Readable via `core.get_setting("boilerplate_min_atom_count")` +- Lower = more aggressive stripping (e.g. `3`); higher = more conservative (e.g. `10`) + +--- + +## Testing / Validation Plan + +### Unit tests — `crates/atomic-core/src/boilerplate.rs` + +```rust +#[test] +fn test_normalize_strips_heading_markers() { ... } + +#[test] +fn test_normalize_collapses_whitespace() { ... } + +#[test] +fn test_content_hash_deterministic() { ... } + +#[test] +fn test_boilerplate_indices_all_unique() { + // All counts < threshold → no indices returned +} + +#[test] +fn test_boilerplate_indices_shared_chunks() { + // 3 chunks, 2 appear in >= 5 atoms → indices {0, 2} returned +} + +#[test] +fn test_boilerplate_indices_fallback_all_boilerplate() { + // All chunks are boilerplate → returns empty set (fallback) +} +``` + +### Integration test — `crates/atomic-core/tests/health_tests.rs` + +```rust +#[tokio::test] +async fn test_boilerplate_chunks_excluded_from_vec_search() { + // 1. Create 6 atoms all sharing the same header chunk + // 2. Run embedding pipeline for all 6 + // 3. Verify: atom_chunks contains the shared header for each atom + // 4. Verify: vec_chunks does NOT contain vectors for the shared header chunks + // 5. Verify: vec_chunks DOES contain vectors for the unique body chunks +} + +#[tokio::test] +async fn test_reembed_strips_boilerplate_retroactively() { + // 1. Create 6 atoms, embed without boilerplate filtering (pre-migration state) + // 2. Trigger retry_embedding on one of the atoms + // 3. Verify shared header chunk's vec_chunks entry is deleted +} + +#[tokio::test] +async fn test_boilerplate_below_threshold_not_stripped() { + // 1. Create 4 atoms (< 5) sharing a header + // 2. Embed all 4 + // 3. Verify shared header IS in vec_chunks (below threshold) +} +``` + +Verification commands: +```bash +cargo test -p atomic-core -- boilerplate +cargo test -p atomic-core -- health +cargo check -p atomic-core -p atomic-server +npx tsc --noEmit +``` + +--- + +## Risks, Assumptions, and Open Questions + +| # | Risk / Assumption | Severity | Mitigation | +|---|-------------------|----------|------------| +| 1 | Backfill of `content_hash` for large DBs may be slow | Medium | Run in background task, not at request time | +| 2 | Threshold of 5 may strip legitimately shared content (e.g. a wiki-style infobox used in exactly 5 articles) | Low | Make configurable; default conservative | +| 3 | After stripping, atoms with 100% boilerplate content get zero semantic vectors — they disappear from search | Medium | Fallback: if all chunks filtered, embed all (already in plan) | +| 4 | `sha2` crate may not be in workspace dependencies | Low | Check `Cargo.toml`; fallback to `ring` if already present | +| 5 | The `delete after save` approach creates a brief window where boilerplate chunks have vectors | Negligible | Single-atom pipeline is synchronous; window is sub-millisecond | +| 6 | Postgres backend (`storage/postgres/chunks.rs`) also needs the same changes | Medium | Mirror all new methods in Postgres implementation | + +**Open question:** Should the health check `boilerplate_pollution` score improve automatically once boilerplate chunks are stripped from `vec_chunks`? Yes — the check queries `semantic_edges WHERE similarity_score >= 0.99`. After re-embedding, similarity scores for these atoms should drop below 0.99 for non-boilerplate content, removing them from the query results. + +--- + +## LOE / Effort Estimate + +| Phase | Task | Hours | +|-------|------|-------| +| 0 | `boilerplate.rs` module | 2h | +| 1 | Schema migration (V11) | 1h | +| 2 | Store `content_hash` on save | 1h | +| 3 | Storage helper: count occurrences | 1h | +| 4 | Inject filtering: single-atom path | 1.5h | +| 5 | Inject filtering: re-embed batch path | 1.5h | +| 6 | Backfill task | 0.5h | +| 7 | UX update (Re-embed button) | 0.5h | +| Tests | Unit + integration | 2h | +| Postgres parity | Mirror new methods | 1.5h | +| **Total** | | **~12.5h** | + +--- + +## Decision Log + +| Date | Decision | Rationale | +|------|----------|-----------| +| 2026-05-01 | Strip before embedding, keep in `atom_chunks` | Preserves FTS, display, and stored atom content intact | +| 2026-05-01 | Add `content_hash` column vs. full-text comparison | Hash index is orders of magnitude faster than full-text equality scan | +| 2026-05-01 | Threshold = 5 atoms (configurable) | Conservative default; avoids stripping shared stylistic choices in small corpora | +| 2026-05-01 | "Delete after save" for vec_chunks | Minimal blast radius vs. changing `save_chunks_and_embeddings_sync` signature | +| 2026-05-01 | Fallback: embed all if all chunks are boilerplate | Prevents atoms from becoming invisible in semantic search | diff --git a/package-lock.json b/package-lock.json index 3629b85b..9694866f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -76,6 +76,8 @@ "@capacitor/cli": "^8.3.0", "@tailwindcss/vite": "^4.0.0", "@tauri-apps/cli": "^2.0.0", + "@testing-library/react": "^16.3.2", + "@testing-library/user-event": "^14.6.1", "@types/d3-force": "^3.0.10", "@types/qrcode": "^1.5.6", "@types/react": "^18.3.0", @@ -4874,6 +4876,77 @@ "@tauri-apps/api": "^2.10.1" } }, + "node_modules/@testing-library/dom": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", + "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/code-frame": "^7.10.4", + "@babel/runtime": "^7.12.5", + "@types/aria-query": "^5.0.1", + "aria-query": "5.3.0", + "dom-accessibility-api": "^0.5.9", + "lz-string": "^1.5.0", + "picocolors": "1.1.1", + "pretty-format": "^27.0.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@testing-library/react": { + "version": "16.3.2", + "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-16.3.2.tgz", + "integrity": "sha512-XU5/SytQM+ykqMnAnvB2umaJNIOsLF3PVv//1Ew4CTcpz0/BRyy/af40qqrt7SjKpDdT1saBMc42CUok5gaw+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.12.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@testing-library/dom": "^10.0.0", + "@types/react": "^18.0.0 || ^19.0.0", + "@types/react-dom": "^18.0.0 || ^19.0.0", + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@testing-library/user-event": { + "version": "14.6.1", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.1.tgz", + "integrity": "sha512-vq7fv0rnt+QTXgPxr5Hjc210p6YKq2kmdziLgnsZGgLJ9e6VAShx1pACLuRjd/AS/sr7phAR58OIIpf0LlmQNw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12", + "npm": ">=6" + }, + "peerDependencies": { + "@testing-library/dom": ">=7.21.4" + } + }, + "node_modules/@types/aria-query": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", + "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -5413,6 +5486,17 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/aria-query": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", + "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "dequal": "^2.0.3" + } + }, "node_modules/array-buffer-byte-length": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.2.tgz", @@ -6652,6 +6736,14 @@ "integrity": "sha512-qiSlmBq9+BCdCA/L46dw8Uy93mloxsPSbwnm5yrKn2vMPiy8KyAskTF6zuV/j5BMsmOGZDPs7KjU+mjb670kfA==", "license": "MIT" }, + "node_modules/dom-accessibility-api": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", + "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -9185,6 +9277,17 @@ "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/lz-string": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz", + "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "lz-string": "bin/bin.js" + } + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -10926,6 +11029,47 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pretty-format": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", + "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "ansi-regex": "^5.0.1", + "ansi-styles": "^5.0.0", + "react-is": "^17.0.1" + }, + "engines": { + "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -11146,6 +11290,14 @@ "react": "^18.3.1" } }, + "node_modules/react-is": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", + "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/react-markdown": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-10.1.0.tgz", diff --git a/package.json b/package.json index b88fd06a..d49010b2 100644 --- a/package.json +++ b/package.json @@ -122,6 +122,8 @@ "@capacitor/cli": "^8.3.0", "@tailwindcss/vite": "^4.0.0", "@tauri-apps/cli": "^2.0.0", + "@testing-library/react": "^16.3.2", + "@testing-library/user-event": "^14.6.1", "@types/d3-force": "^3.0.10", "@types/qrcode": "^1.5.6", "@types/react": "^18.3.0", diff --git a/src/components/dashboard/widgets/HealthReviewModal.tsx b/src/components/dashboard/widgets/HealthReviewModal.tsx index 7000efd2..1d8e0784 100644 --- a/src/components/dashboard/widgets/HealthReviewModal.tsx +++ b/src/components/dashboard/widgets/HealthReviewModal.tsx @@ -2,7 +2,7 @@ import { useState, useEffect, useCallback } from 'react'; import { createPortal } from 'react-dom'; import { X, GitMerge, Link, Loader2, CheckCircle, - ChevronDown, ChevronUp, ExternalLink, RefreshCw, + ChevronDown, ChevronUp, RefreshCw, } from 'lucide-react'; import { getTransport } from '../../../lib/transport'; @@ -26,12 +26,6 @@ interface AtomDetail { type PairAction = 'merge_with_llm' | 'keep_both'; type PairStatus = 'idle' | 'loading' | 'done' | 'error'; -// Atom preview (content_quality + boilerplate) -interface AtomPreview { - id: string; - title: string; - created_at?: string; -} // Boilerplate atom entry interface BoilerplateEntry { @@ -252,9 +246,10 @@ function BoilerplateSection({ atoms }: { atoms: BoilerplateEntry[] }) {

    Embedding quality issue

    - These {atoms.length} atom{atoms.length !== 1 ? 's' : ''} share identical boilerplate sections - that dominate their embeddings — semantic search cannot reliably distinguish them from - each other. Edit each atom to remove or uniquify the boilerplate sections, then re-embed. + These {atoms.length} atom{atoms.length !== 1 ? 's' : ''} share identical boilerplate + sections that dominate their embeddings. Re-embedding will automatically strip the + shared sections from the semantic index while preserving your original content. + After re-embedding, run a fresh health check to see the updated score.

    @@ -279,7 +274,7 @@ function BoilerplateSection({ atoms }: { atoms: BoilerplateEntry[] }) {
    {status === 'done' ? ( - Queued + Re-queued — boilerplate will be stripped ) : status === 'error' ? ( Failed @@ -293,7 +288,7 @@ function BoilerplateSection({ atoms }: { atoms: BoilerplateEntry[] }) { {status === 'loading' ? : } - Re-embed + Strip & re-embed )}
    @@ -558,7 +553,7 @@ export function HealthReviewModal({ report, checkName, onClose, onResolved }: Pr const tabs = [ ...(overlapPairs.length > 0 ? [{ key: 'content_overlap', label: 'Content overlap', count: overlapPairs.length }] : []), - ...(boilerplateAtoms.length > 0 ? [{ key: 'boilerplate', label: 'Boilerplate', count: boilerplateAtoms.length }] : []), + ...(boilerplateAtoms.length > 0 ? [{ key: 'boilerplate_pollution', label: 'Boilerplate', count: boilerplateAtoms.length }] : []), ...(contradictionCount > 0 ? [{ key: 'contradiction_detection', label: 'Contradictions', count: contradictionCount }] : []), ...(noSourceCount > 0 ? [{ key: 'content_quality', label: 'No source', count: noSourceCount }] : []), ...(rootlessCount > 0 ? [{ key: 'tag_health', label: 'Tag structure', count: rootlessCount }] : []), @@ -658,7 +653,7 @@ export function HealthReviewModal({ report, checkName, onClose, onResolved }: Pr )} - {activeTab === 'boilerplate' && ( + {activeTab === 'boilerplate_pollution' && ( )} diff --git a/src/components/dashboard/widgets/HealthWidget.tsx b/src/components/dashboard/widgets/HealthWidget.tsx index f936f95f..c9856199 100644 --- a/src/components/dashboard/widgets/HealthWidget.tsx +++ b/src/components/dashboard/widgets/HealthWidget.tsx @@ -127,7 +127,7 @@ const CHECK_DESCRIPTIONS: Record) => stri }, boilerplate_pollution: (d) => { const count = d.count as number ?? 0; - return `${count} atom${count !== 1 ? 's' : ''} share so much template text that semantic search can't distinguish them`; + return `${count} atom${count !== 1 ? 's' : ''} share identical boilerplate text that drowns out their unique content in embeddings — click Re-embed to strip it from the semantic index`; }, broken_internal_links: (d) => { const n = (d.broken_count as number) ?? 0; diff --git a/src/components/dashboard/widgets/__tests__/HealthReviewModal.test.tsx b/src/components/dashboard/widgets/__tests__/HealthReviewModal.test.tsx index 6f4d594c..c505a533 100644 --- a/src/components/dashboard/widgets/__tests__/HealthReviewModal.test.tsx +++ b/src/components/dashboard/widgets/__tests__/HealthReviewModal.test.tsx @@ -110,7 +110,7 @@ describe('HealthReviewModal', () => { onResolved={onResolved} /> ); - expect(screen.getByText('Content overlap')).toBeTruthy(); + // 'Content overlap' tab label only appears when >1 tab; check content instead expect(screen.getByText('Article Alpha')).toBeTruthy(); expect(screen.getByText('Article Beta')).toBeTruthy(); expect(screen.getByText('72% overlap')).toBeTruthy(); @@ -132,7 +132,7 @@ describe('HealthReviewModal', () => { render( @@ -169,7 +169,7 @@ describe('HealthReviewModal', () => { onResolved={onResolved} /> ); - expect(screen.getByText('Contradictions')).toBeTruthy(); + // 'Contradictions' tab label only appears when >1 tab; check content instead expect(screen.getByText('Topic X Version 1')).toBeTruthy(); expect(screen.getByText('Topic X Version 2')).toBeTruthy(); expect(screen.getByText(/85% similarity/)).toBeTruthy(); @@ -199,7 +199,7 @@ describe('HealthReviewModal', () => { onResolved={onResolved} /> ); - expect(screen.getByText('No source')).toBeTruthy(); + // 'No source' tab label only appears when >1 tab; check content instead expect(screen.getByText('Note Without Source')).toBeTruthy(); expect(screen.getByText('Another Unsourced Note')).toBeTruthy(); expect(screen.getByText(/1\/15\/2026|Jan 15|15 Jan/)).toBeTruthy(); @@ -226,7 +226,7 @@ describe('HealthReviewModal', () => { onResolved={onResolved} /> ); - expect(screen.getByText('Tag structure')).toBeTruthy(); + // Tab bar only shows with >1 tab; content still renders expect(screen.getByText('Orphaned Category')).toBeTruthy(); expect(screen.getByText('Floating Topic')).toBeTruthy(); expect(screen.getByText(/7 atom/)).toBeTruthy(); @@ -260,7 +260,7 @@ describe('HealthReviewModal', () => { render( diff --git a/tsconfig.json b/tsconfig.json index 34cad748..168af12b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,7 +21,7 @@ "noFallthroughCasesInSwitch": true }, "include": ["src"], - "exclude": ["src/**/__tests__/**", "src/**/*.test.ts", "src/**/*.test.tsx"] + "exclude": ["src/**/__tests__/**", "src/**/*.test.ts", "src/**/*.test.tsx"], "references": [{ "path": "./tsconfig.node.json" }] } From d5736a91858df9301599a7eba0077145d5a92dce Mon Sep 17 00:00:00 2001 From: bk-ty Date: Fri, 1 May 2026 14:07:09 -0500 Subject: [PATCH 04/51] =?UTF-8?q?feat:=20Review=20Queue=20v2=20Phase=20A?= =?UTF-8?q?=20=E2=80=94=20inline=20actions=20on=20NoSource,=20Tag,=20Boile?= =?UTF-8?q?rplate=20tabs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add shared types/applyFix helper in review/types.ts - NoSourceRow: add_source URL input + mark_intentional dismissal - TagRootlessRow: move_under parent select + dismiss - BoilerplateAtomRow: reembed button with status feedback - Refactor BoilerplateSection, ContentQualitySection, TagHealthSection to use new row components with per-section local removal state (items disappear on resolve) - TagHealthSection uses useTagsStore to populate parent dropdown - Add afterEach(cleanup) to NoSourceRow tests to prevent DOM bleed - All 14 tests pass, tsc --noEmit clean --- crates/atomic-core/src/db.rs | 28 +- crates/atomic-core/src/health/mod.rs | 135 +++++- crates/atomic-core/src/storage/mod.rs | 80 ++++ .../atomic-core/src/storage/sqlite/health.rs | 96 ++++ crates/atomic-server/src/routes/health.rs | 17 +- docs/plans/2026-05-01-review-queue-v2/plan.md | 452 ++++++++++++++++++ .../dashboard/widgets/HealthReviewModal.tsx | 179 +++---- .../dashboard/widgets/HealthWidget.tsx | 2 +- .../__tests__/BoilerplateAtomRow.test.tsx | 29 ++ .../widgets/__tests__/NoSourceRow.test.tsx | 64 +++ .../widgets/__tests__/TagRootlessRow.test.tsx | 36 ++ .../widgets/review/BoilerplateAtomRow.tsx | 56 +++ .../dashboard/widgets/review/NoSourceRow.tsx | 122 +++++ .../widgets/review/TagRootlessRow.tsx | 95 ++++ .../dashboard/widgets/review/types.ts | 56 +++ 15 files changed, 1325 insertions(+), 122 deletions(-) create mode 100644 docs/plans/2026-05-01-review-queue-v2/plan.md create mode 100644 src/components/dashboard/widgets/__tests__/BoilerplateAtomRow.test.tsx create mode 100644 src/components/dashboard/widgets/__tests__/NoSourceRow.test.tsx create mode 100644 src/components/dashboard/widgets/__tests__/TagRootlessRow.test.tsx create mode 100644 src/components/dashboard/widgets/review/BoilerplateAtomRow.tsx create mode 100644 src/components/dashboard/widgets/review/NoSourceRow.tsx create mode 100644 src/components/dashboard/widgets/review/TagRootlessRow.tsx create mode 100644 src/components/dashboard/widgets/review/types.ts diff --git a/crates/atomic-core/src/db.rs b/crates/atomic-core/src/db.rs index 9d7b8b6f..8bfde4bb 100644 --- a/crates/atomic-core/src/db.rs +++ b/crates/atomic-core/src/db.rs @@ -211,7 +211,7 @@ impl Database { /// 1. Add a new `if version < N` block at the end (before the virtual-table section) /// 2. End the block with `PRAGMA user_version = N;` /// 3. Bump LATEST_VERSION - const LATEST_VERSION: i32 = 17; + const LATEST_VERSION: i32 = 18; pub fn run_migrations(conn: &Connection) -> Result<(), AtomicCoreError> { Self::run_migrations_internal(conn, false) @@ -858,9 +858,14 @@ impl Database { // --- V16 → V17: content_hash column on atom_chunks for boilerplate detection --- if version < 17 { + // ALTER TABLE ADD COLUMN has no IF NOT EXISTS in SQLite; ignore the error + // if the column was already added (e.g. during a test migration re-run). + let _ = conn.execute( + "ALTER TABLE atom_chunks ADD COLUMN content_hash TEXT", + [], + ); conn.execute_batch( r#" - ALTER TABLE atom_chunks ADD COLUMN content_hash TEXT; CREATE INDEX IF NOT EXISTS idx_atom_chunks_content_hash ON atom_chunks(content_hash); PRAGMA user_version = 17; @@ -868,6 +873,25 @@ impl Database { )?; } + // --- V17 → V18: persistent dismissals for the review queue --- + if version < 18 { + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS health_dismissals ( + id TEXT PRIMARY KEY, + check_name TEXT NOT NULL, + item_key TEXT NOT NULL, + reason TEXT NOT NULL, + dismissed_at TEXT NOT NULL, + expires_at TEXT + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_health_dismissals_lookup + ON health_dismissals(check_name, item_key); + PRAGMA user_version = 18; + "#, + )?; + } + // --- Triggers (recreated every startup to stay current) --- conn.execute_batch( "DROP TRIGGER IF EXISTS atom_tags_insert_count; diff --git a/crates/atomic-core/src/health/mod.rs b/crates/atomic-core/src/health/mod.rs index f772d22b..edae1aa2 100644 --- a/crates/atomic-core/src/health/mod.rs +++ b/crates/atomic-core/src/health/mod.rs @@ -315,6 +315,21 @@ pub async fn compute_health(core: &AtomicCore) -> Result = + dismissed_pairs.into_iter().map(|(k, _)| k).collect(); + if let Some(result) = checks.get_mut(check_name) { + apply_dismissals(check_name, result, &dismissed); + } + } + + // Aggregate score let overall_score = aggregate_score(&checks); let overall_status = HealthStatus::from_score(overall_score).as_str().to_string(); @@ -370,7 +385,7 @@ pub async fn compute_single_check( core: &AtomicCore, check_name: &str, ) -> Result<(String, HealthCheckResult), AtomicCoreError> { - let result = match check_name { + let mut result = match check_name { // Async check — requires per-atom DB lookups "broken_internal_links" => compute_link_check(core).await?, // Sync checks — fetch raw data once, dispatch to the appropriate fn @@ -407,6 +422,15 @@ pub async fn compute_single_check( ))) } }; + // Apply persistent dismissals + if matches!(check_name, "content_overlap" | "contradiction_detection" | "boilerplate_pollution" | "content_quality" | "tag_health") { + let dismissed_pairs = core.storage().list_dismissed_keys_sync(check_name).await.unwrap_or_default(); + if !dismissed_pairs.is_empty() { + let dismissed: std::collections::HashSet = + dismissed_pairs.into_iter().map(|(k, _)| k).collect(); + apply_dismissals(check_name, &mut result, &dismissed); + } + } Ok((check_name.to_string(), result)) } @@ -707,5 +731,114 @@ pub async fn run_fix( } + + +/// Build a stable item key for a pair. Sorts atom IDs lexicographically so +/// key ordering is independent of which atom is A vs B. +pub fn pair_key(a: &str, b: &str) -> String { + if a <= b { + format!("{}__{}", a, b) + } else { + format!("{}__{}", b, a) + } +} + +/// Filter a check result's JSON data to exclude dismissed entries. +pub(crate) fn apply_dismissals( + check_name: &str, + result: &mut HealthCheckResult, + dismissed_keys: &std::collections::HashSet, +) { + if dismissed_keys.is_empty() { + return; + } + + use serde_json::Value; + let data = &mut result.data; + + match check_name { + "content_overlap" => { + if let Some(pairs) = data.get_mut("pairs").and_then(Value::as_array_mut) { + pairs.retain(|p| { + let a = p.get("atom_a").and_then(|o| o.get("id")).and_then(Value::as_str).unwrap_or(""); + let b = p.get("atom_b").and_then(|o| o.get("id")).and_then(Value::as_str).unwrap_or(""); + !dismissed_keys.contains(&pair_key(a, b)) + }); + let new_count = pairs.len(); + if let Some(c) = data.get_mut("count") { + *c = Value::from(new_count); + } + if let Some(c) = data.get_mut("cross_source_overlaps") { + *c = Value::from(new_count); + } + } + } + "contradiction_detection" => { + if let Some(pairs) = data.get_mut("pairs").and_then(Value::as_array_mut) { + pairs.retain(|p| { + let a = p.get("atom_a").and_then(|o| o.get("id")).and_then(Value::as_str).unwrap_or(""); + let b = p.get("atom_b").and_then(|o| o.get("id")).and_then(Value::as_str).unwrap_or(""); + !dismissed_keys.contains(&pair_key(a, b)) + }); + let new_count = pairs.len(); + if let Some(c) = data.get_mut("potential_contradictions") { + *c = Value::from(new_count); + } + if new_count == 0 { + result.requires_review = false; + } + } + } + "boilerplate_pollution" => { + if let Some(arr) = data.get_mut("affected_atoms").and_then(Value::as_array_mut) { + arr.retain(|entry| { + let id = entry.get("id").and_then(Value::as_str).unwrap_or(""); + !dismissed_keys.contains(id) + }); + let new_count = arr.len(); + if let Some(c) = data.get_mut("count") { + *c = Value::from(new_count); + } + if new_count == 0 { + result.requires_review = false; + } + } + } + "content_quality" => { + if let Some(ns) = data + .pointer_mut("/issues/no_source/atoms") + .and_then(Value::as_array_mut) + { + ns.retain(|entry| { + let id = entry.get("id").and_then(Value::as_str).unwrap_or(""); + !dismissed_keys.contains(id) + }); + let new_count = ns.len(); + if let Some(c) = data.pointer_mut("/issues/no_source/count") { + *c = Value::from(new_count); + } + if new_count == 0 { + result.requires_review = false; + } + } + } + "tag_health" => { + if let Some(arr) = data.get_mut("rootless_tag_list").and_then(Value::as_array_mut) { + arr.retain(|t| { + let id = t.get("id").and_then(Value::as_str).unwrap_or(""); + !dismissed_keys.contains(id) + }); + let new_count = arr.len(); + if let Some(c) = data.get_mut("rootless_tags") { + *c = Value::from(new_count); + } + if new_count == 0 { + result.requires_review = false; + } + } + } + _ => {} + } +} #[cfg(test)] mod tests; \ No newline at end of file diff --git a/crates/atomic-core/src/storage/mod.rs b/crates/atomic-core/src/storage/mod.rs index 585baa8d..479d085c 100644 --- a/crates/atomic-core/src/storage/mod.rs +++ b/crates/atomic-core/src/storage/mod.rs @@ -332,6 +332,86 @@ impl StorageBackend { StorageBackend::Postgres(_) => Ok(None), } } + + pub(crate) async fn get_tag_by_id_sync( + &self, + tag_id: &str, + ) -> Result)>, AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + let tag_id = tag_id.to_string(); + tokio::task::spawn_blocking(move || s.get_tag_by_id_impl(&tag_id)) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(None), + } + } + + pub(crate) async fn list_dismissed_keys_sync( + &self, + check_name: &str, + ) -> Result, AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + let check_name = check_name.to_string(); + tokio::task::spawn_blocking(move || s.list_dismissed_keys_impl(&check_name)) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(Vec::new()), + } + } + + pub(crate) async fn dismiss_health_item_sync( + &self, + check_name: &str, + item_key: &str, + reason: &str, + expires_at: Option<&str>, + ) -> Result<(), AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + let check_name = check_name.to_string(); + let item_key = item_key.to_string(); + let reason = reason.to_string(); + let expires_at = expires_at.map(String::from); + tokio::task::spawn_blocking(move || { + s.dismiss_health_item_impl(&check_name, &item_key, &reason, expires_at.as_deref()) + }) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(()), + } + } + + pub(crate) async fn undismiss_health_item_sync( + &self, + check_name: &str, + item_key: &str, + ) -> Result<(), AtomicCoreError> { + match self { + StorageBackend::Sqlite(s) => { + let s = s.clone(); + let check_name = check_name.to_string(); + let item_key = item_key.to_string(); + tokio::task::spawn_blocking(move || { + s.undismiss_health_item_impl(&check_name, &item_key) + }) + .await + .map_err(join_err)? + } + #[cfg(feature = "postgres")] + StorageBackend::Postgres(_) => Ok(()), + } + } pub(crate) async fn count_chunk_hash_occurrences_sync( &self, hashes: &[String], diff --git a/crates/atomic-core/src/storage/sqlite/health.rs b/crates/atomic-core/src/storage/sqlite/health.rs index 35ce0ed9..519332ce 100644 --- a/crates/atomic-core/src/storage/sqlite/health.rs +++ b/crates/atomic-core/src/storage/sqlite/health.rs @@ -870,4 +870,100 @@ pub(crate) fn source_prefix(url: &Option) -> String { return u[..slash].to_string(); } u.clone() +} + +// ==================== Dismissal methods ==================== + +impl SqliteStorage { + /// Get a tag by ID. Returns (name, parent_id). + pub(crate) fn get_tag_by_id_impl( + &self, + tag_id: &str, + ) -> Result)>, AtomicCoreError> { + let conn = self.db.read_conn()?; + let mut stmt = conn.prepare( + "SELECT name, parent_id FROM tags WHERE id = ?1", + )?; + let result = stmt + .query_map(params![tag_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + })? + .next() + .transpose()?; + Ok(result) + } + + /// List currently active dismissals for a check. Returns (item_key, reason) pairs. + pub(crate) fn list_dismissed_keys_impl( + &self, + check_name: &str, + ) -> Result, AtomicCoreError> { + let conn = self.db.read_conn()?; + let now = chrono::Utc::now().to_rfc3339(); + let mut stmt = conn.prepare( + "SELECT item_key, reason FROM health_dismissals + WHERE check_name = ?1 + AND (expires_at IS NULL OR expires_at > ?2)", + )?; + let rows = stmt + .query_map(params![check_name, now], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()?; + Ok(rows) + } + + /// Insert or update a dismissal (upsert on unique index). + pub(crate) fn dismiss_health_item_impl( + &self, + check_name: &str, + item_key: &str, + reason: &str, + expires_at: Option<&str>, + ) -> Result<(), AtomicCoreError> { + let mut conn = self + .db + .conn + .lock() + .map_err(|e| AtomicCoreError::Lock(e.to_string()))?; + let id = uuid::Uuid::new_v4().to_string(); + let now = chrono::Utc::now().to_rfc3339(); + let tx = conn + .transaction() + .map_err(|e| AtomicCoreError::DatabaseOperation(e.to_string()))?; + tx.execute( + "INSERT INTO health_dismissals (id, check_name, item_key, reason, dismissed_at, expires_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(check_name, item_key) DO UPDATE SET + reason = excluded.reason, + dismissed_at = excluded.dismissed_at, + expires_at = excluded.expires_at", + params![id, check_name, item_key, reason, now, expires_at], + )?; + tx.commit() + .map_err(|e| AtomicCoreError::DatabaseOperation(e.to_string()))?; + Ok(()) + } + + pub(crate) fn undismiss_health_item_impl( + &self, + check_name: &str, + item_key: &str, + ) -> Result<(), AtomicCoreError> { + let mut conn = self + .db + .conn + .lock() + .map_err(|e| AtomicCoreError::Lock(e.to_string()))?; + let tx = conn + .transaction() + .map_err(|e| AtomicCoreError::DatabaseOperation(e.to_string()))?; + tx.execute( + "DELETE FROM health_dismissals WHERE check_name = ?1 AND item_key = ?2", + params![check_name, item_key], + )?; + tx.commit() + .map_err(|e| AtomicCoreError::DatabaseOperation(e.to_string()))?; + Ok(()) + } } \ No newline at end of file diff --git a/crates/atomic-server/src/routes/health.rs b/crates/atomic-server/src/routes/health.rs index 999155e2..6a1a6938 100644 --- a/crates/atomic-server/src/routes/health.rs +++ b/crates/atomic-server/src/routes/health.rs @@ -9,8 +9,9 @@ use crate::db_extractor::Db; use actix_web::{web, HttpResponse}; +use atomic_core::compaction; use atomic_core::health::{ - self, audit, FixRequest, FixResponse, HealthCheckResult, HealthReport, + self, audit, pair_key, FixRequest, FixResponse, HealthCheckResult, HealthReport, }; use atomic_core::health::audit::{HealthFixLog, StoredHealthReport}; use serde::{Deserialize, Serialize}; @@ -19,12 +20,16 @@ use utoipa::ToSchema; /// Request body for the per-item fix endpoint. #[derive(Deserialize, Serialize, ToSchema)] pub struct ManualFixRequest { - /// "merge", "keep_both", "delete_one", "merge_with_llm" pub action: String, - /// For merge/delete operations: which atom to keep. - pub keep_atom_id: Option, - /// "keep_newer", "keep_longer", "llm" - pub merge_strategy: Option, + // Optional per-action fields + pub url: Option, + pub parent_id: Option, + pub into_tag_id: Option, + pub content: Option, + pub winner_atom_id: Option, + pub loser_atom_id: Option, + #[serde(default)] + pub dry_run: bool, } /// Query params for history endpoint. diff --git a/docs/plans/2026-05-01-review-queue-v2/plan.md b/docs/plans/2026-05-01-review-queue-v2/plan.md new file mode 100644 index 00000000..6e6aedff --- /dev/null +++ b/docs/plans/2026-05-01-review-queue-v2/plan.md @@ -0,0 +1,452 @@ +# Knowledge Health Review Queue — UI Improvements (v2) + +**Date:** 2026-05-01 +**Status:** Planning +**Project:** Atomic +**Request:** Enhance the existing Review Queue modal with per-item inline actions, per-tab re-scan, richer resolution workflow (3-option Keep A/Keep B/Merge, source/recency badges, diff highlighting), batch selection, filtering/sorting, resolved counters, markdown export, and dashboard deep-linking. Preserve existing theme and layout. + +--- + +## Executive Summary + +The Review Queue modal (`HealthReviewModal.tsx`) renders 5 tabs over a single `HealthReport` snapshot. Today it has lightweight actions on two tabs (Content overlap: Merge/Keep both; Boilerplate: informational) and passive display on the other three (Contradictions, No source, Tag structure). The proposed v2 turns it into an interactive queue: every tab supports per-item actions, batch selection, filtering, and a persistent "dismissed/resolved" state. Several backend additions are required — dismissal storage, per-item source updates, per-check re-scan, LLM strip-boilerplate and merge-editor previews, and tag merge/move endpoints exposed for the modal. + +**Recommended phasing:** ship in 4 waves — (A) dismissals + inline actions that reuse existing endpoints, (B) per-tab re-scan + resolved counters + lazy loading, (C) resolution upgrades (3-option resolver, diff highlighting, source badges, merge editor), (D) batch operations + export + dashboard integration. + +--- + +## Current Architecture & Evidence + +### Modal structure — `src/components/dashboard/widgets/HealthReviewModal.tsx` (644 lines) + +- Single top-level `HealthReviewModal` component (L502–L643) takes a full `HealthReport` and a `checkName` pre-selector +- Tabs array built at L520–L526 — included conditionally based on which `checks[*]` has non-empty data +- `selectedTab` state at L528; `activeTab` defaults to first available tab (L529) +- `resolvedCount` state (L531) currently only increments on Content overlap `applyPairFix`; no persistence across sessions or tabs +- Escape key + body scroll lock at L533–L541; no other keyboard shortcuts +- `applyPairFix` callback at L543–L556 calls `apply_health_item_fix` with `check: 'duplicate_detection'`; `setResolvedCount(n => n + 1)` on success +- Tab bodies: `PairRow` (L68–L197), `BoilerplateSection` (L227–L267), `ContradictionRow`/`ContradictionSection` (L271–L376), `ContentQualitySection` (L380–L427), `TagHealthSection` (L431–L487) +- No batch selection, no filtering, no sorting controls, no re-scan, no export, no source/recency badges, no diff highlighting + +### Action endpoints — `crates/atomic-server/src/routes/health.rs` + +| Endpoint | What it does today | Relevance | +|---|---|---| +| `GET /api/health/knowledge` | Full report with all 5 review-data blobs | Used on modal open | +| `POST /api/health/fix` (`run_health_fix`) | Batch auto-fix across all checks | Used by the dashboard's big button, not by the modal | +| `POST /api/health/fix/{check}/{item_id}` (`apply_manual_fix`) | Per-item manual fix | Currently only handles `(duplicate_detection, merge_with_llm)` (L100–L125); **all other check+action pairs return 400** | +| `POST /api/health/undo/{fix_id}` | Undo a logged fix | Wired in the dashboard undo toast | +| `GET /api/health/history` / `GET /api/health/fixes/recent` | Historical reports and fix log | Not used by the modal | +| `POST /api/health/check/{check_name}` (`compute_single_check`) | Re-run one check in isolation | **Already exists** — can power per-tab re-scan | + +### Existing fix primitives we can reuse + +- `crates/atomic-core/src/health/llm_fixes.rs` + - `merge_duplicate_pair(core, atom_a, atom_b, dry_run)` — returns the merged content when `dry_run=true` (no writes), otherwise writes + logs (L79–L226). Already supports preview. + - `fix_untagged_complete_atoms(core, ids, dry_run)` — re-runs tagging pipeline +- `crates/atomic-core/src/storage/sqlite/tags.rs` + - `apply_tag_merges_impl(&[TagMerge { winner_name, loser_name, reason }])` — canonical tag merge path (L512–L532); also exposed on `AtomicCore::apply_tag_merges` (`lib.rs` L2134) + - `update_tag_impl(id, name, parent_id)` at L178–L219 — can reparent a tag (used for "Move under…") + - `delete_tag_impl(id, recursive)` at L394 — exists for orphan cleanup +- Atom updates: `update_atom` command (command-map.ts L74–L79) takes `{ content, source_url, published_at, tag_ids, ... }` — "Add source" inline can reuse this with the atom's existing content + +### Data shapes that back the queue + +All live inside the `HealthReport` blob computed on demand; per-atom pre-fetches happen in `PairRow.toggleExpand` (L96–L110) and `ContradictionRow.toggleExpand` (L276–L292) via `get_atom`. There is **no** persistent "dismissed" state — if you refresh, everything that was dismissed returns. + +### Dashboard integration — `src/components/dashboard/widgets/HealthWidget.tsx` + +- "Apply N automatic fixes" button at the bottom of the widget (excluded checks tracked in `excludedFromFix`, L370+). Already has tooltip/label infrastructure. +- `setShowReviewModal(checkName)` is called from two places — auto-pops to first `requires_review` check on the main "Review" button, and from the `HealthCheckRow` component per-row. Deep-link works already; the issue is post-resolution dashboard refresh. + +--- + +## Recommended Approach + +Split across 4 phases so UX improvements land incrementally and each wave is independently shippable: + +| Phase | Theme | Major deps | +|---|---|---| +| **A** | Dismissals + inline per-item actions | New DB table `health_dismissals`; extend `apply_manual_fix` | +| **B** | Per-tab re-scan, resolved counters, lazy content fetch | Reuse `compute_single_check`; new `checkUpdatedAt` state | +| **C** | 3-option resolver, source/recency badges, diff highlighting, merge-editor, contradiction summary | Extend `get_atom` cache; LLM-powered conflict summary; `diff-match-patch` dep | +| **D** | Batch selection, Strip boilerplate LLM pass, export, dashboard real-time sync | New `POST /api/health/strip-boilerplate`; frontend markdown export helper | + +### Why dismissals must be persistent + +Every feature (resolved counter, "Show deferred" toggle, "Mark intentional", batch dismiss, "Ignore pair") depends on somewhere to store *"this item should not appear until the underlying condition changes"*. Without it, every refresh re-surfaces everything, which defeats the queue metaphor. The cheapest fix is a new `health_dismissals` table keyed by `(check_name, item_key)` — see Phase A below for the schema. + +### Dependency footprint + +- Backend (Rust): 1 new migration, 1 new table, ~6 new endpoints/wrapper methods, 1 LLM prompt for "strip boilerplate", 1 LLM prompt for "contradiction summary" +- Frontend (TS): `diff-match-patch` (≈ 50KB, widely used, no peer deps). All other work uses existing primitives (Zustand store, Tailwind, lucide icons) + +--- + +## Implementation Plan + +### Phase A — Dismissals + Inline Per-Item Actions (~20h) + +#### A1. New `health_dismissals` table (migration V18) + +**File:** `crates/atomic-core/src/db.rs` + +```sql +CREATE TABLE IF NOT EXISTS health_dismissals ( + id TEXT PRIMARY KEY, + check_name TEXT NOT NULL, + item_key TEXT NOT NULL, -- e.g. atom_id, pair_id, tag_id, 'a_b' for pairs + reason TEXT NOT NULL, -- 'intentional_no_source', 'ignored_pair', 'deferred', 'resolved_other' + dismissed_at TEXT NOT NULL, + expires_at TEXT -- null = permanent until underlying data changes +); +CREATE UNIQUE INDEX idx_health_dismissals_lookup + ON health_dismissals(check_name, item_key); +``` + +Bump `LATEST_VERSION` to 18. Follow the V17 idempotent pattern if existing tests re-run the migration. + +#### A2. Storage methods and `AtomicCore` wrappers + +**File:** `crates/atomic-core/src/storage/sqlite/health.rs` + +```rust +pub(crate) fn list_dismissed_keys_impl(&self, check_name: &str) -> StorageResult>; +pub(crate) fn dismiss_health_item_impl(&self, check_name: &str, item_key: &str, reason: &str, expires_at: Option<&str>) -> StorageResult<()>; +pub(crate) fn undismiss_health_item_impl(&self, check_name: &str, item_key: &str) -> StorageResult<()>; +``` + +Wire through `StorageBackend` (async) and `AtomicCore`. + +#### A3. Filter dismissed items inside `compute_single_check` / `compute_health` + +Add one `SELECT` per reviewable check. Feed a `HashSet` of dismissed keys into the check function and exclude matches from `data.pairs` / `data.affected_atoms` / `data.issues.no_source.atoms` / `data.rootless_tag_list`. + +Keep item keys stable: +- `content_overlap` / `contradiction_detection`: `{atom_a_id}__{atom_b_id}` sorted by id lexicographically +- `content_quality` no_source: atom_id +- `boilerplate_pollution`: atom_id +- `tag_health` rootless: tag_id +- `tag_health` similar_name: `{winner_id}__{loser_id}` sorted + +#### A4. Extend `apply_manual_fix` with new (check, action) tuples + +**File:** `crates/atomic-server/src/routes/health.rs` (L93–L126) + +| check | action | Body | Behaviour | +|---|---|---|---| +| `content_overlap` | `keep_a` / `keep_b` | — | Delete loser atom; log undoable `before_state` | +| `content_overlap` | `dismiss` | — | Insert dismissal reason=`resolved_other` | +| `contradiction_detection` | `defer` | — | Dismissal with `expires_at = now + 7 days` | +| `contradiction_detection` | `dismiss` | — | Dismissal `resolved_other` | +| `contradiction_detection` | `summary` | — | LLM one-liner (Phase C4) | +| `content_quality` | `add_source` | `{url}` | `update_atom` preserving existing content | +| `content_quality` | `mark_intentional` | — | Dismissal reason=`intentional_no_source` | +| `tag_health` | `move_under` | `{parent_id}` | `update_tag_impl(id, name, Some(parent_id))` | +| `tag_health` | `merge` | `{into_tag_id}` | `apply_tag_merges_impl` | +| `tag_health` | `ignore_pair` | — | Dismissal `ignored_pair` | +| `boilerplate_pollution` | `reembed` | — | Enqueue `retry_embedding` | + +#### A5. Frontend: per-item actions + +- **No source tab**: `NoSourceRow` component — inline URL input + Save; "Mark intentional"; "Open ↗" +- **Tag structure tab**: rootless rows get "Move under…" dropdown populated from `useTags()` store; similar-name pairs get "Merge" confirm dialog + "Ignore pair" +- **Boilerplate tab**: "View edges" lazy-expand (same pattern as `PairRow.toggleExpand`); "Re-embed" button; "Strip boilerplate" disabled with "Coming soon" tooltip until Phase D + +--- + +### Phase B — Per-Tab Re-scan, Resolved Counters, Lazy Loading (~10h) + +#### B1. Per-tab "Re-scan" button + +Top of each tab body: `↻ Re-scan`. Calls `health_check_single({check_name})` (command already exists at command-map.ts L722). On success, splice result into local `report.checks[name]` state. + +Track `lastScannedAt: Record`; render "Last checked: 2m ago" via `Intl.RelativeTimeFormat`. + +#### B2. Resolved counters + +Upgrade `resolvedCount` to `Record`. Persist to localStorage scoped by active database id. Clear daily. Show "Resolved today: N" at the top of each tab; add a progress bar (`X / initial_queue_size`). + +#### B3. Lazy tab content + virtualization + +Only mount the active tab's body. For >50 items in a tab, wrap the list in `@tanstack/react-virtual` — already in deps via the canvas widget. + +--- + +### Phase C — Resolution Upgrades (~25h) + +#### C1. 3-option resolver for pairs + +Replace Merge/Keep both with `Keep A | Keep B | Merge (edit)`: + +- `Keep A` / `Keep B` — archive the loser via new `apply_manual_fix` action +- `Merge (edit)` — opens `MergeEditorModal`: + 1. Call `apply_health_item_fix` with `action: merge_with_llm, dry_run: true` (already supported by `merge_duplicate_pair` at llm_fixes.rs L79+) + 2. Show synthesis in CodeMirror editor pre-populated with dry-run content + 3. "Save merge" → new action `merge_with_edited_content` body `{ content, winner_atom_id, loser_atom_id }` + 4. Single `FixAction` for undo + +#### C2. Source trust + recency indicator + +Backend: content-overlap SQL already joins atoms; add `created_at` to the selected columns in `storage/sqlite/health.rs`. Contradiction query needs the same enrichment. + +Frontend helper: +```ts +function trustScore(source: string | null, createdAt: string): { badge: string; score: number } +``` +- +10 if hostname is in the `trusted_sources` setting (comma-separated) +- +5 if `created_at` within last 30 days +- Render per-atom badge; higher-scoring atom gets "Recommended" chip; ties → no chip + +#### C3. Diff highlighting + +Add `diff-match-patch` (~50KB). In `PairRow` / `ContradictionRow` expanded view, replace raw `
    ` with line-diff. Atom A pane highlights removals red; Atom B pane highlights additions green. Content always fully visible.
+
+#### C4. Contradiction summary (LLM)
+
+New action in `apply_manual_fix`: `(contradiction_detection, summary)` body empty. Calls LLM: "In one sentence describe what factual claims conflict between these atoms, or 'no real conflict' if the differences are perspective, not fact." Cache per `pair_id` in frontend state.
+
+#### C5. "Flag for later" + "Show deferred"
+
+`defer` action inserts dismissal with `expires_at = now + 7d`. Tab header shows `Show deferred (N)` toggle when any deferred items exist. When enabled, pass `?include_deferred=true` to `compute_single_check` — rename the param if it conflicts, otherwise add it to the query extractor.
+
+---
+
+### Phase D — Batch, Strip Boilerplate, Export, Dashboard Sync (~25h)
+
+#### D1. Selection mode
+
+Checkbox per row. State `selectedItems: Record>` keyed by tab. Floating action bar when any selected:
+
+```
+[3 selected]  [Dismiss all]  [Apply suggested merge]  [Clear]
+```
+
+Sequential batch dispatch with progress callback. Undo stack captures all action ids; Undo applies in reverse.
+
+#### D2. "Strip boilerplate" LLM pass
+
+**New endpoint:** `POST /api/health/strip-boilerplate/{atom_id}` body `{ dry_run: bool }`.
+
+New function `strip_boilerplate` in `llm_fixes.rs`:
+1. Load atom + all atoms sharing ≥5 near-identical chunks (via `semantic_edges` ≥0.99)
+2. LLM prompt: "The following atoms share template text. Return the unique content of atom_X only, preserving its specific details but removing shared sections present in all samples."
+3. `dry_run=true` returns proposed content; `false` writes via `update_atom_content_only`
+
+Frontend: dry-run → before/after diff modal → confirm → real call.
+
+#### D3. Export queue to Markdown
+
+Frontend-only. New `buildReviewQueueMarkdown(report, dismissals)` that iterates all 5 tab datasets and emits the format the prompt specifies. Reuse web/Tauri file-save split from existing `HealthExportModal.tsx`. Button lives in the modal header.
+
+#### D4. Dashboard real-time sync
+
+Debounce-wrap `fetchHealth()` in `HealthPanel` so batch actions only trigger one refresh. Optional: backend returns `{ dirty: true }` on dismissal changes so the dashboard can show "Scores may be stale — refresh" instead of forcing recompute.
+
+#### D5. "Apply N automatic fixes" tooltip
+
+Add tooltip `"Auto-fixes only affect: broken links, re-tagging empty atoms, trimming long content. Manual review items are handled in the Review Queue."` to the button in `HealthWidget.tsx`.
+
+---
+
+## Files / Components To Change
+
+### Backend (Rust)
+
+| File | Change |
+|---|---|
+| `crates/atomic-core/src/db.rs` | V18 migration; bump `LATEST_VERSION`; idempotent ALTER pattern |
+| `crates/atomic-core/src/storage/sqlite/health.rs` | 3 dismissal methods; enrich overlap/contradiction queries with `created_at` |
+| `crates/atomic-core/src/storage/mod.rs` | `StorageBackend` async wrappers for the 3 dismissal methods |
+| `crates/atomic-core/src/health/checks.rs` | Thread `dismissed_keys` into every reviewable check; exclude matching items |
+| `crates/atomic-core/src/health/mod.rs` | `compute_health` / `compute_single_check` pass dismissals; add `include_deferred` param |
+| `crates/atomic-core/src/health/llm_fixes.rs` | New `strip_boilerplate` function; new `merge_with_edited_content`; `summarize_contradiction` |
+| `crates/atomic-server/src/routes/health.rs` | Extend `apply_manual_fix` match with all new action tuples; add `POST /api/health/strip-boilerplate/{atom_id}` + OpenAPI annotation; thread `include_deferred` query param through `compute_single_check` |
+| `crates/atomic-server/src/routes/mod.rs` | Register new strip-boilerplate route |
+| `crates/atomic-server/src/lib.rs` | Add new handler + schema types to `#[openapi(paths(...))]` |
+
+### Frontend (TypeScript)
+
+| File | Change |
+|---|---|
+| `src/components/dashboard/widgets/HealthReviewModal.tsx` | Split into multiple files; add checkbox state, sort/filter bar, export button, per-tab re-scan |
+| `src/components/dashboard/widgets/review/NoSourceRow.tsx` | **New** — inline URL editor + Mark intentional + Open |
+| `src/components/dashboard/widgets/review/TagRootlessRow.tsx` | **New** — Move under dropdown + Dismiss |
+| `src/components/dashboard/widgets/review/TagSimilarPairRow.tsx` | **New** — Merge confirm + Ignore pair |
+| `src/components/dashboard/widgets/review/BoilerplateAtomRow.tsx` | **New** — View edges expand + Re-embed |
+| `src/components/dashboard/widgets/review/MergeEditorModal.tsx` | **New** — CodeMirror merge editor with dry-run pre-fill |
+| `src/components/dashboard/widgets/review/PairDiffView.tsx` | **New** — diff-match-patch line-mode rendering for side-by-side |
+| `src/components/dashboard/widgets/review/ReviewQueueExport.ts` | **New** — buildReviewQueueMarkdown helper |
+| `src/components/dashboard/widgets/review/trustScore.ts` | **New** — source/recency scoring helper |
+| `src/components/dashboard/widgets/HealthWidget.tsx` | Debounced refresh on `onResolved`; "Apply N fixes" tooltip |
+| `src/lib/transport/command-map.ts` | `strip_health_boilerplate` entry; pass `include_deferred` to `health_check_single` |
+| `package.json` | Add `diff-match-patch` + `@types/diff-match-patch` |
+
+---
+
+## Data Flow / Interfaces
+
+### Dismissal lifecycle
+
+```
+user clicks "Mark intentional"
+  → POST /api/health/fix/content_quality/{atom_id}  body {action: "mark_intentional"}
+  → dismiss_health_item(check="content_quality", key=atom_id, reason="intentional_no_source")
+  → frontend optimistically removes row; onResolved() fires
+  → next compute_single_check() excludes this atom until it gains a source URL
+```
+
+### Merge-editor flow
+
+```
+user clicks "Merge (edit)" on a pair
+  → POST /api/health/fix/content_overlap/{pair_id}  body {action: "merge_with_llm", dry_run: true}
+  → merge_duplicate_pair(dry_run=true) returns synthesized content (no writes)
+  → MergeEditorModal opens; pre-fills CodeMirror with synthesis
+  → user edits, clicks "Save merge"
+  → POST /api/health/fix/content_overlap/{pair_id}  body {action: "merge_with_edited_content", content, winner_atom_id, loser_atom_id}
+  → update_atom(winner.id, edited_content); delete_atom(loser.id); log FixAction
+```
+
+### Batch dispatch
+
+```
+user selects 3 pairs, clicks "Dismiss all"
+  → for each: POST /api/health/fix/content_overlap/{pair_id} {action: "dismiss"}
+  → frontend shows "Processing 2/3…"
+  → on completion: toast "✅ 3 pairs dismissed" with Undo
+  → Undo → reverse sequence of undismiss calls
+```
+
+---
+
+## Configuration / Secrets / Deployment Notes
+
+### New settings (optional — default empty/off)
+
+- `trusted_sources` (string, comma-separated hostnames) — used by trustScore helper
+- `review_queue.auto_defer_days` (int, default 7) — expires_at for "Flag for later"
+- `review_queue.batch_concurrency` (int, default 1) — parallelism for batch dispatch; keep at 1 by default to preserve order for undo
+
+No secrets needed. No new env vars. The LLM endpoints reuse the already-configured provider (OpenRouter or Ollama).
+
+### Schema migration deployment
+
+The V18 migration is additive-only (new table + unique index). Safe to deploy without downtime. Backfill is not required — an empty `health_dismissals` table means nothing is dismissed, which is the correct initial state.
+
+### OpenAPI surface
+
+Register 1 new path (`/api/health/strip-boilerplate/{atom_id}`) plus extended request body schema for `ManualFixRequest` (add optional fields for `url`, `parent_id`, `into_tag_id`, `content`, `winner_atom_id`, `loser_atom_id`, `dry_run`). All schemas under `#[cfg_attr(feature = "openapi", derive(ToSchema))]` matching the existing convention.
+
+---
+
+## Testing / Validation Plan
+
+### Automated
+
+**Backend unit tests** (`crates/atomic-core/src/health/tests.rs`):
+- `test_dismissed_content_overlap_excluded` — create fixture with 3 pairs, dismiss 1, confirm `compute_single_check` returns 2
+- `test_dismissed_tag_health_rootless_excluded` — same pattern for rootless tags
+- `test_contradiction_defer_expires` — insert dismissal with `expires_at` in the past, confirm item reappears
+- `test_add_source_updates_atom` — call `apply_manual_fix` with `add_source`, verify atom `source_url` is set without touching content
+- `test_move_under_reparents_tag` — `apply_manual_fix` with `move_under`, verify `update_tag_impl` was called with the new parent
+- `test_tag_merge_via_health_fix` — `apply_manual_fix` with `merge`, verify `apply_tag_merges_impl` ran and atoms were re-tagged
+- `test_keep_a_archives_b` — confirm loser atom is soft-deleted, winner untouched
+- `test_merge_dry_run_returns_content_no_writes` — already covered by existing `merge_duplicate_pair` test; add assertion that atom row count unchanged
+- `test_strip_boilerplate_dry_run` — stub LLM, confirm original atom content unchanged after dry_run
+
+**Frontend unit tests** (`src/components/dashboard/widgets/__tests__/`):
+- `NoSourceRow.test.tsx` — click Add source, enter URL, verify `update_atom` called with correct body
+- `TagRootlessRow.test.tsx` — select parent from dropdown, verify `apply_health_item_fix` with `move_under`
+- `MergeEditorModal.test.tsx` — mock dry_run response, render editor, edit content, save, verify final mutation
+- `PairDiffView.test.tsx` — snapshot test for red/green highlighting of a known diff
+- `trustScore.test.ts` — table-driven cases (trusted hostname wins, recent age beats old, ties produce no chip)
+- `ReviewQueueExport.test.ts` — fixture report → matches expected markdown byte-for-byte
+
+**Commands:**
+```bash
+cargo test -p atomic-core -- health
+cargo test -p atomic-core -- boilerplate
+cargo test -p atomic-server -- health
+cargo check -p atomic-core -p atomic-server
+npx tsc --noEmit
+npx vitest run src/components/dashboard/widgets/__tests__/
+npm run lint
+```
+
+### Manual / E2E
+
+- Build the desktop app: `npm run tauri dev`
+- Seed 3 overlapping atoms, 2 no-source atoms, 2 rootless tags in a test DB
+- Exercise each per-item action; verify dismissed items stay dismissed across modal close/reopen
+- Trigger batch dismiss on 3 items; verify undo rolls all 3 back
+- Trigger Merge-edit flow; confirm the editor pre-fills and saving updates both atoms
+- Run `npm run build:mobile` then load in Capacitor iOS/Android to smoke-test the new touch-friendly controls (checkboxes, inline URL input). Capacitor builds may require the simulator running — use `npm run dev:mobile:ios` for a live loop.
+- Export queue to markdown; diff against a known-good fixture
+
+### Blockers for runnable E2E
+- The project does not appear to ship a Playwright / Cypress harness in the repo — E2E is manual via `npm run tauri dev`. If the team wants automated E2E, that's a separate track and not covered here.
+- LLM-dependent features (Merge, Strip boilerplate, Contradiction summary) require a reachable OpenRouter key or a running Ollama instance. Tests should mock the provider via the existing `MockLlmProvider` pattern (search support/mod.rs) to avoid hitting the network.
+
+---
+
+## Risks, Assumptions, and Open Questions
+
+### Risks
+
+| Risk | Severity | Mitigation |
+|---|---|---|
+| Dismissal table grows unbounded (every dismissed pair, tag, atom) | Medium | Add periodic cleanup: delete dismissals where underlying atom/tag no longer exists; cap total at 10k rows per check with FIFO eviction |
+| Merge-edit flow can race: user A dismisses while user B is mid-merge | Low | Idempotent actions: merge_with_edited_content checks both atoms still exist before writing; returns 409 on conflict |
+| LLM cost for Contradiction summary × 20 pairs on modal open | Medium | Lazy-fetch: only call summary when user expands a pair; cache per-session |
+| Diff-match-patch for very long atoms (>10k chars) is slow | Low | Truncate content to first 2000 chars for diff view; show "Content truncated for diff; click Open to view full atom" |
+| Batch dispatch partial failure (3rd of 5 fails) | Medium | Stop on first failure; show error toast with what was applied; user can retry the rest |
+| "Strip boilerplate" LLM hallucination removes unique content | High | Always dry-run first; show before/after diff; never auto-apply in batch |
+| Move-under dropdown performance with 1000s of tags | Low | Virtualize the dropdown (same library as tag tree) |
+
+### Assumptions
+
+- `HealthReport.checks[*].data` structure is stable across all five reviewable checks (verified in current code)
+- `merge_duplicate_pair` dry-run returns content in a predictable shape — verify in `llm_fixes.rs`; may need adapter
+- `update_atom` preserves `created_at` / `updated_at` semantics (verify; should it bump `updated_at` on source-only edit?)
+- Tag merge UI does not need to preview affected atoms before applying — a count is sufficient. If the team wants full preview, add a separate "Preview merge" dry-run mode to `apply_tag_merges_impl`
+- `retry_embedding` command is the correct primitive for the "Re-embed" button; already exists in command-map (verify exact name)
+
+### Open questions
+
+1. **Archived vs deleted for Keep A / Keep B** — do we soft-archive (set a `status` flag) or hard-delete? Current `delete_atom` hard-deletes. Soft-archive would need schema work. Recommendation: use existing hard-delete; rely on `health_fix_log.before_state` snapshot for undo. Check whether `log_fix` already stores atom content snapshot on delete.
+2. **Item-key collision across DBs** — if the same atom_id exists in two databases, dismissal needs DB scoping. `health_dismissals` is per-DB (lives in data DB, not registry DB), so this is implicit — verify when wiring the migration.
+3. **Where does the "resolved today" counter live?** — localStorage is simple but doesn't sync across devices. Server-side is more work. Recommendation: localStorage for Phase B; revisit if users want cross-device.
+4. **What happens to a dismissed content_overlap pair when one of the atoms is deleted?** — dismissal becomes stale. Cleanup job: delete `health_dismissals` rows whose `item_key` references a non-existent atom. Run on startup + weekly.
+5. **Strip boilerplate threshold** — ≥5 shared edges is the current boilerplate-detection threshold. Reuse that, or make it configurable per-call?
+6. **Tag merge confirm count** — `count_atoms_with_tags` already exists. Just wire it up.
+
+---
+
+## LOE / Effort Estimate
+
+| Phase | Hours | Deliverable |
+|---|---|---|
+| A | 20 | Dismissals table + per-item actions on all tabs (except Strip/Merge-edit) |
+| B | 10 | Re-scan + resolved counters + lazy content |
+| C | 25 | 3-option resolver + merge editor + source badges + diff highlighting + contradiction summary + "Flag for later" |
+| D | 25 | Batch selection + Strip boilerplate LLM pass + Export + Dashboard sync + tooltip |
+| **Total** | **80** | ~2.5 weeks at 30 hrs/week |
+
+Additive: ~10% (8h) for unit + component tests across all phases, split evenly. ~15% (12h) for integration/manual QA across mobile + desktop.
+
+**Net estimate:** ~100 hours end-to-end, matching the complexity of the original v1 health dashboard plan.
+
+---
+
+## Decision Log
+
+| Date | Decision | Rationale |
+|---|---|---|
+| 2026-05-01 | New `health_dismissals` table rather than per-check flag columns | Single polymorphic table serves all 5 checks; easier to add future review categories |
+| 2026-05-01 | Dismissal keys are string composites (`a__b` sorted) | Avoids schema changes when we add new key shapes; frontend can construct keys without backend knowledge |
+| 2026-05-01 | Reuse `merge_duplicate_pair` dry-run for merge-editor pre-fill | Primitive already exists; avoids new LLM code paths |
+| 2026-05-01 | Diff-match-patch over a richer diff library (e.g., react-diff-viewer) | 50KB vs 200KB+; we only need line-mode highlighting, not a full diff UI |
+| 2026-05-01 | Resolved counter in localStorage, not server-side | Low stakes; avoids new storage roundtrips. Revisit if cross-device sync requested |
+| 2026-05-01 | Hard-delete for Keep A/Keep B, rely on `before_state` for undo | Avoids soft-delete schema work; existing undo infra already handles this pattern for `fix_source_uniqueness` |
+| 2026-05-01 | Phase ordering A→B→C→D | Each phase is independently shippable; A unlocks everything, D has the most LLM cost and lowest UX criticality |
diff --git a/src/components/dashboard/widgets/HealthReviewModal.tsx b/src/components/dashboard/widgets/HealthReviewModal.tsx
index 1d8e0784..72a85b1d 100644
--- a/src/components/dashboard/widgets/HealthReviewModal.tsx
+++ b/src/components/dashboard/widgets/HealthReviewModal.tsx
@@ -1,10 +1,14 @@
-import { useState, useEffect, useCallback } from 'react';
+import { useState, useEffect, useCallback, useMemo } from 'react';
 import { createPortal } from 'react-dom';
 import {
   X, GitMerge, Link, Loader2, CheckCircle,
-  ChevronDown, ChevronUp, RefreshCw,
+  ChevronDown, ChevronUp,
 } from 'lucide-react';
 import { getTransport } from '../../../lib/transport';
+import { useTagsStore } from '../../../stores/tags';
+import { NoSourceRow } from './review/NoSourceRow';
+import { TagRootlessRow } from './review/TagRootlessRow';
+import { BoilerplateAtomRow } from './review/BoilerplateAtomRow';
 
 // ==================== Types ====================
 
@@ -224,21 +228,17 @@ function ActionBtn({
 
 // ==================== Boilerplate section ====================
 
-function BoilerplateSection({ atoms }: { atoms: BoilerplateEntry[] }) {
-  const [reembedStatus, setReembedStatus] = useState>({});
+function BoilerplateSection({ atoms, onResolved }: { atoms: BoilerplateEntry[]; onResolved: () => void }) {
+  const [removed, setRemoved] = useState>(new Set());
+  const visible = atoms.filter(a => !removed.has(a.id));
 
-  const reembed = async (atomId: string) => {
-    setReembedStatus(prev => ({ ...prev, [atomId]: 'loading' }));
-    try {
-      await getTransport().invoke('retry_embedding', { atomId });
-      setReembedStatus(prev => ({ ...prev, [atomId]: 'done' }));
-    } catch {
-      setReembedStatus(prev => ({ ...prev, [atomId]: 'error' }));
-    }
+  const handleResolved = (id: string) => {
+    setRemoved(prev => new Set(prev).add(id));
+    onResolved();
   };
 
-  if (atoms.length === 0) {
-    return 
    No boilerplate pollution detected — all clear
    ;
+  if (visible.length === 0) {
+    return 
    No boilerplate pollution — all clear
    ;
   }
 
   return (
@@ -246,55 +246,15 @@ function BoilerplateSection({ atoms }: { atoms: BoilerplateEntry[] }) {
       
    
         
    Embedding quality issue
    
         
    
-          These {atoms.length} atom{atoms.length !== 1 ? 's' : ''} share identical boilerplate
-          sections that dominate their embeddings. Re-embedding will automatically strip the
-          shared sections from the semantic index while preserving your original content.
-          After re-embedding, run a fresh health check to see the updated score.
+          These {visible.length} atom{visible.length !== 1 ? 's' : ''} have near-identical semantic edges.
+          Their unique content is drowned out by shared template text. Edit the atoms to make unique
+          content more prominent, then Re-embed to refresh their vectors.
         
    
       
    
       
    
-        {atoms
-          .slice()
-          .sort((a, b) => b.clone_count - a.clone_count)
-          .map(atom => {
-            const status = reembedStatus[atom.id] ?? 'idle';
-            return (
-              
    
-                
    
-                  
    
-                    {atom.title || Untitled atom}
-                  
    
-                  
    
-                    {atom.clone_count} near-identical edge{atom.clone_count !== 1 ? 's' : ''}
-                  
    
-                
    
-                
    
-                  {status === 'done' ? (
-                    
-                       Re-queued — boilerplate will be stripped
-                    
-                  ) : status === 'error' ? (
-                    Failed
-                  ) : (
-                    
-                  )}
-                
    
-              
    
-            );
-          })}
+        {visible.slice().sort((a, b) => b.clone_count - a.clone_count).map(atom => (
+          
+        ))}
    ); @@ -411,16 +371,22 @@ function ContradictionSection({ data }: { data: Record }) { // ==================== Content quality (no-source) section ==================== -function ContentQualitySection({ data }: { data: Record }) { +function ContentQualitySection({ data, onResolved }: { data: Record; onResolved: () => void }) { const issues = data.issues as Record; }> | undefined; const noSourceItems = (issues?.no_source?.atoms ?? []) as Array<{ id: string; title: string; created_at?: string }>; - const noSourceCount = issues?.no_source?.count ?? noSourceItems.length; + const [removed, setRemoved] = useState>(new Set()); + const visible = noSourceItems.filter(a => !removed.has(a.id)); + + const handleResolved = (id: string) => { + setRemoved(prev => new Set(prev).add(id)); + onResolved(); + }; - if (noSourceCount === 0) { + if (visible.length === 0) { return

    No unsourced atoms — all clear

    ; } @@ -428,32 +394,16 @@ function ContentQualitySection({ data }: { data: Record }) {

    - {noSourceCount} atom{noSourceCount !== 1 ? 's' : ''} missing a source URL + {visible.length} atom{visible.length !== 1 ? 's' : ''} missing a source URL

    - These atoms have no source_url{' '} - and no URL or{' '} - Source: line in their content. - Open each atom in the editor and add a source URL to resolve. + Add a source URL for each, or Mark intentional if the atom doesn’t have one + (e.g. meeting notes, personal writing).

    - {noSourceItems.map(atom => ( -
    -
    -

    - {atom.title || Untitled atom} -

    - {atom.created_at && ( -

    - Created {new Date(atom.created_at).toLocaleDateString()} -

    - )} -
    -
    + {visible.map(atom => ( + ))}
    @@ -462,41 +412,46 @@ function ContentQualitySection({ data }: { data: Record }) { // ==================== Tag health (rootless) section ==================== -function TagHealthSection({ data }: { data: Record }) { +function TagHealthSection({ data, onResolved }: { data: Record; onResolved: () => void }) { const rootlessList = (data.rootless_tag_list as RootlessTag[] | undefined) ?? []; - const rootlessCount = (data.rootless_tags as number) ?? rootlessList.length; const similarCount = (data.similar_name_pairs as number) ?? 0; + const [removed, setRemoved] = useState>(new Set()); + const visible = rootlessList.filter(t => !removed.has(t.id)); + + const allTags = useTagsStore(s => s.tags); + const parentOptions = useMemo(() => { + const rootlessIds = new Set(rootlessList.map(t => t.id)); + return allTags + .filter(t => !rootlessIds.has(t.id)) + .map(t => ({ id: t.id, name: t.name })); + }, [allTags, rootlessList]); + + const handleResolved = (id: string) => { + setRemoved(prev => new Set(prev).add(id)); + onResolved(); + }; return (
    - {rootlessList.length > 0 && ( + {visible.length > 0 && (

    - {rootlessCount} root-level tag{rootlessCount !== 1 ? 's' : ''} with no parent + {visible.length} root-level tag{visible.length !== 1 ? 's' : ''} with no parent

    - These tags sit at the top level. Consider nesting them under a relevant - category to keep the tag tree navigable. + Pick a parent to nest them under, or Dismiss to leave at root.

    - {rootlessList - .slice() - .sort((a, b) => b.atom_count - a.atom_count) - .map(tag => ( -
    -
    -

    {tag.name}

    -

    - {tag.atom_count} atom{tag.atom_count !== 1 ? 's' : ''} -

    -
    -
    - ))} + {visible.slice().sort((a, b) => b.atom_count - a.atom_count).map(tag => ( + + ))}
    )} @@ -507,13 +462,13 @@ function TagHealthSection({ data }: { data: Record }) { {similarCount} similar-name pair{similarCount !== 1 ? 's' : ''}

    - Tags with near-identical names (e.g. "React" and "ReactJS") may be duplicates. - Review and merge in the tag tree if needed. + Tags with near-identical names (e.g. “React” and “ReactJS”) may be duplicates. + Review and merge from the tag tree if needed. (Inline merge coming in Phase C.)

    )} - {rootlessList.length === 0 && similarCount === 0 && ( + {visible.length === 0 && similarCount === 0 && (

    Tag structure is healthy — all clear

    )}
    @@ -654,7 +609,7 @@ export function HealthReviewModal({ report, checkName, onClose, onResolved }: Pr )} {activeTab === 'boilerplate_pollution' && ( - + setResolvedCount(n => n + 1)} /> )} {activeTab === 'contradiction_detection' && contradictionData && ( @@ -662,11 +617,11 @@ export function HealthReviewModal({ report, checkName, onClose, onResolved }: Pr )} {activeTab === 'content_quality' && contentQualityData && ( - + setResolvedCount(n => n + 1)} /> )} {activeTab === 'tag_health' && tagHealthData && ( - + setResolvedCount(n => n + 1)} /> )}
    diff --git a/src/components/dashboard/widgets/HealthWidget.tsx b/src/components/dashboard/widgets/HealthWidget.tsx index c9856199..3772e1ac 100644 --- a/src/components/dashboard/widgets/HealthWidget.tsx +++ b/src/components/dashboard/widgets/HealthWidget.tsx @@ -127,7 +127,7 @@ const CHECK_DESCRIPTIONS: Record) => stri }, boilerplate_pollution: (d) => { const count = d.count as number ?? 0; - return `${count} atom${count !== 1 ? 's' : ''} share identical boilerplate text that drowns out their unique content in embeddings — click Re-embed to strip it from the semantic index`; + return `${count} atom${count !== 1 ? 's' : ''} have near-identical semantic edges — their embeddings can’t be distinguished in search. Usually caused by shared template structure in the content.`; }, broken_internal_links: (d) => { const n = (d.broken_count as number) ?? 0; diff --git a/src/components/dashboard/widgets/__tests__/BoilerplateAtomRow.test.tsx b/src/components/dashboard/widgets/__tests__/BoilerplateAtomRow.test.tsx new file mode 100644 index 00000000..fb201512 --- /dev/null +++ b/src/components/dashboard/widgets/__tests__/BoilerplateAtomRow.test.tsx @@ -0,0 +1,29 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { BoilerplateAtomRow } from '../review/BoilerplateAtomRow'; + +const invoke = vi.fn(); +vi.mock('../../../../lib/transport', () => ({ + getTransport: () => ({ invoke }), +})); + +describe('BoilerplateAtomRow', () => { + beforeEach(() => { + invoke.mockReset(); + invoke.mockResolvedValue({ status: 'ok' }); + }); + + it('triggers re-embed', async () => { + const onResolved = vi.fn(); + const user = userEvent.setup(); + render(); + await user.click(screen.getByText('Re-embed')); + await waitFor(() => expect(invoke).toHaveBeenCalledWith('apply_health_item_fix', expect.objectContaining({ + check: 'boilerplate_pollution', + item_id: 'a1', + action: 'reembed', + }))); + await waitFor(() => expect(onResolved).toHaveBeenCalledWith('a1'), { timeout: 1000 }); + }); +}); diff --git a/src/components/dashboard/widgets/__tests__/NoSourceRow.test.tsx b/src/components/dashboard/widgets/__tests__/NoSourceRow.test.tsx new file mode 100644 index 00000000..184ba56e --- /dev/null +++ b/src/components/dashboard/widgets/__tests__/NoSourceRow.test.tsx @@ -0,0 +1,64 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen, waitFor, cleanup } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { NoSourceRow } from '../review/NoSourceRow'; + +const invoke = vi.fn(); +vi.mock('../../../../lib/transport', () => ({ + getTransport: () => ({ invoke }), +})); + +describe('NoSourceRow', () => { + beforeEach(() => { + invoke.mockReset(); + invoke.mockResolvedValue({ status: 'ok' }); + }); + afterEach(() => cleanup()); + + const atom = { id: 'a1', title: 'Meeting Notes', created_at: '2026-03-01T00:00:00Z' }; + + it('renders title + date', () => { + render( {}} />); + expect(screen.getByText('Meeting Notes')).toBeTruthy(); + expect(screen.getByText(/Created/i)).toBeTruthy(); + }); + + it('saves a source URL via apply_health_item_fix', async () => { + const onResolved = vi.fn(); + const user = userEvent.setup(); + render(); + await user.click(screen.getByText('Add source')); + const input = screen.getByPlaceholderText('https://\u2026') as HTMLInputElement; + await user.type(input, 'https://example.com'); + await user.click(screen.getByText('Save')); + await waitFor(() => expect(invoke).toHaveBeenCalledWith('apply_health_item_fix', expect.objectContaining({ + check: 'content_quality', + item_id: 'a1', + action: 'add_source', + url: 'https://example.com', + }))); + await waitFor(() => expect(onResolved).toHaveBeenCalledWith('a1'), { timeout: 1000 }); + }); + + it('marks intentional', async () => { + const onResolved = vi.fn(); + const user = userEvent.setup(); + render(); + await user.click(screen.getByText('Intentional')); + await waitFor(() => expect(invoke).toHaveBeenCalledWith('apply_health_item_fix', expect.objectContaining({ + check: 'content_quality', + item_id: 'a1', + action: 'mark_intentional', + }))); + }); + + it('shows error when save fails', async () => { + invoke.mockRejectedValueOnce(new Error('nope')); + const user = userEvent.setup(); + render( {}} />); + await user.click(screen.getByText('Add source')); + await user.type(screen.getByPlaceholderText('https://\u2026'), 'x'); + await user.click(screen.getByText('Save')); + await waitFor(() => expect(screen.getByText('nope')).toBeTruthy()); + }); +}); diff --git a/src/components/dashboard/widgets/__tests__/TagRootlessRow.test.tsx b/src/components/dashboard/widgets/__tests__/TagRootlessRow.test.tsx new file mode 100644 index 00000000..057040d3 --- /dev/null +++ b/src/components/dashboard/widgets/__tests__/TagRootlessRow.test.tsx @@ -0,0 +1,36 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { TagRootlessRow } from '../review/TagRootlessRow'; + +const invoke = vi.fn(); +vi.mock('../../../../lib/transport', () => ({ + getTransport: () => ({ invoke }), +})); + +describe('TagRootlessRow', () => { + beforeEach(() => { + invoke.mockReset(); + invoke.mockResolvedValue({ status: 'ok' }); + }); + + const tag = { id: 't1', name: 'Foo', atom_count: 3 }; + const parents = [{ id: 'p1', name: 'Topics' }, { id: 'p2', name: 'People' }]; + + it('moves tag under selected parent', async () => { + const onResolved = vi.fn(); + const user = userEvent.setup(); + render(); + await user.selectOptions(screen.getByRole('combobox'), 'p1'); + const buttons = screen.getAllByRole('button'); + // Move button is the one that is not "Leave at root" dismiss + const moveBtn = buttons.find(b => !b.hasAttribute('disabled') && b.getAttribute('title') !== "Leave at root \u2014 won't be flagged again"); + await user.click(moveBtn!); + await waitFor(() => expect(invoke).toHaveBeenCalledWith('apply_health_item_fix', expect.objectContaining({ + check: 'tag_health', + item_id: 't1', + action: 'move_under', + parent_id: 'p1', + }))); + }); +}); diff --git a/src/components/dashboard/widgets/review/BoilerplateAtomRow.tsx b/src/components/dashboard/widgets/review/BoilerplateAtomRow.tsx new file mode 100644 index 00000000..4c3e8a4a --- /dev/null +++ b/src/components/dashboard/widgets/review/BoilerplateAtomRow.tsx @@ -0,0 +1,56 @@ +import { useState } from 'react'; +import { RefreshCw, Loader2, Check } from 'lucide-react'; +import { applyFix, type BoilerplateEntry, type ItemStatus } from './types'; + +export interface BoilerplateAtomRowProps { + atom: BoilerplateEntry; + onResolved: (atomId: string) => void; +} + +export function BoilerplateAtomRow({ atom, onResolved }: BoilerplateAtomRowProps) { + const [status, setStatus] = useState('idle'); + const [error, setError] = useState(null); + + const reembed = async () => { + setStatus('saving'); + setError(null); + try { + await applyFix('boilerplate_pollution', atom.id, { action: 'reembed' }); + setStatus('done'); + setTimeout(() => onResolved(atom.id), 400); + } catch (e) { + setStatus('error'); + setError(e instanceof Error ? e.message : 'Failed to re-embed'); + } + }; + + return ( +
    +
    +
    +

    + {atom.title || Untitled atom} +

    +

    + {atom.clone_count} near-identical edge{atom.clone_count !== 1 ? 's' : ''} +

    +
    + +
    + {error &&

    {error}

    } +
    + ); +} diff --git a/src/components/dashboard/widgets/review/NoSourceRow.tsx b/src/components/dashboard/widgets/review/NoSourceRow.tsx new file mode 100644 index 00000000..7b35d6c2 --- /dev/null +++ b/src/components/dashboard/widgets/review/NoSourceRow.tsx @@ -0,0 +1,122 @@ +import { useState } from 'react'; +import { ExternalLink, Check, Loader2, EyeOff } from 'lucide-react'; +import type { AtomPreview, ItemStatus } from './types'; +import { applyFix } from './types'; + +export interface NoSourceRowProps { + atom: AtomPreview; + onResolved: (atomId: string) => void; +} + +export function NoSourceRow({ atom, onResolved }: NoSourceRowProps) { + const [editing, setEditing] = useState(false); + const [url, setUrl] = useState(''); + const [status, setStatus] = useState('idle'); + const [error, setError] = useState(null); + + const save = async () => { + const trimmed = url.trim(); + if (!trimmed) { + setError('Enter a URL'); + return; + } + setStatus('saving'); + setError(null); + try { + await applyFix('content_quality', atom.id, { action: 'add_source', url: trimmed }); + setStatus('done'); + setTimeout(() => onResolved(atom.id), 400); + } catch (e) { + setStatus('error'); + setError(e instanceof Error ? e.message : 'Failed to save'); + } + }; + + const dismiss = async () => { + setStatus('saving'); + setError(null); + try { + await applyFix('content_quality', atom.id, { action: 'mark_intentional' }); + setStatus('done'); + setTimeout(() => onResolved(atom.id), 400); + } catch (e) { + setStatus('error'); + setError(e instanceof Error ? e.message : 'Failed to dismiss'); + } + }; + + const openAtom = () => { + window.dispatchEvent(new CustomEvent('app-open-atom', { detail: { atomId: atom.id } })); + }; + + return ( +
    +
    +
    +

    + {atom.title || Untitled atom} +

    + {atom.created_at && ( +

    + Created {new Date(atom.created_at).toLocaleDateString()} +

    + )} +
    +
    + + + +
    +
    + + {editing && ( +
    + setUrl(e.target.value)} + onKeyDown={e => { if (e.key === 'Enter') void save(); }} + placeholder="https://…" + autoFocus + className="flex-1 bg-[#161616] border border-white/10 rounded px-2 py-1 text-xs text-gray-200 focus:outline-none focus:border-purple-500" + /> + +
    + )} + + {error &&

    {error}

    } +
    + ); +} diff --git a/src/components/dashboard/widgets/review/TagRootlessRow.tsx b/src/components/dashboard/widgets/review/TagRootlessRow.tsx new file mode 100644 index 00000000..562ab309 --- /dev/null +++ b/src/components/dashboard/widgets/review/TagRootlessRow.tsx @@ -0,0 +1,95 @@ +import { useState, useMemo } from 'react'; +import { EyeOff, Loader2, Check } from 'lucide-react'; +import { applyFix, type RootlessTag, type ItemStatus } from './types'; + +interface TagOption { id: string; name: string; } + +export interface TagRootlessRowProps { + tag: RootlessTag; + parentOptions: TagOption[]; + onResolved: (tagId: string) => void; +} + +export function TagRootlessRow({ tag, parentOptions, onResolved }: TagRootlessRowProps) { + const [parentId, setParentId] = useState(''); + const [status, setStatus] = useState('idle'); + const [error, setError] = useState(null); + + const options = useMemo( + () => parentOptions.filter(o => o.id !== tag.id), + [parentOptions, tag.id], + ); + + const move = async () => { + if (!parentId) { + setError('Pick a parent tag'); + return; + } + setStatus('saving'); + setError(null); + try { + await applyFix('tag_health', tag.id, { action: 'move_under', parent_id: parentId }); + setStatus('done'); + setTimeout(() => onResolved(tag.id), 400); + } catch (e) { + setStatus('error'); + setError(e instanceof Error ? e.message : 'Failed to move tag'); + } + }; + + const dismiss = async () => { + setStatus('saving'); + setError(null); + try { + await applyFix('tag_health', tag.id, { action: 'dismiss' }); + setStatus('done'); + setTimeout(() => onResolved(tag.id), 400); + } catch (e) { + setStatus('error'); + setError(e instanceof Error ? e.message : 'Failed to dismiss'); + } + }; + + return ( +
    +
    +
    +

    {tag.name}

    +

    + {tag.atom_count} atom{tag.atom_count !== 1 ? 's' : ''} +

    +
    +
    + + + +
    +
    + {error &&

    {error}

    } +
    + ); +} diff --git a/src/components/dashboard/widgets/review/types.ts b/src/components/dashboard/widgets/review/types.ts new file mode 100644 index 00000000..837d3053 --- /dev/null +++ b/src/components/dashboard/widgets/review/types.ts @@ -0,0 +1,56 @@ +export interface OverlapPair { + pair_id: string; + atom_a: { id: string; title: string; source?: string; created_at?: string }; + atom_b: { id: string; title: string; source?: string; created_at?: string }; + similarity: number; + shared_tag_count: number; + available_actions?: string[]; +} + +export interface AtomPreview { + id: string; + title: string; + created_at?: string; +} + +export interface BoilerplateEntry { + id: string; + title: string; + clone_count: number; +} + +export interface ContradictionPair { + pair_id: string; + atom_a: { id: string; title: string; source?: string }; + atom_b: { id: string; title: string; source?: string }; + similarity: number; + shared_tag_count: number; +} + +export interface RootlessTag { + id: string; + name: string; + atom_count: number; +} + +export type ItemStatus = 'idle' | 'saving' | 'done' | 'error'; + +/// Build a stable pair key matching the backend's pair_key helper. +export function pairKey(a: string, b: string): string { + return a <= b ? `${a}__${b}` : `${b}__${a}`; +} + +export function applyFix( + check: string, + itemId: string, + body: Record, +) { + // Lazy import to avoid circular + return import('../../../../lib/transport').then(({ getTransport }) => + getTransport().invoke('apply_health_item_fix', { + check, + item_id: itemId, + ...body, + }), + ); +} From fe7f8fb006f7dccfe5c422a990574a48b012ac19 Mon Sep 17 00:00:00 2001 From: bk-ty Date: Fri, 1 May 2026 14:26:20 -0500 Subject: [PATCH 05/51] =?UTF-8?q?feat(health):=20Phase=20B=20=E2=80=94=20p?= =?UTF-8?q?er-tab=20re-scan,=20resolved=20counters,=20virtualized=20overla?= =?UTF-8?q?p=20list?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add local report state with re-sync effect so Re-scan can mutate per-tab - Add rescanTab() calling health_check_single and splicing result into state - Add TabHeader subcomponent: Re-scan button, last-scanned timestamp (relative), resolved-today counter with progress bar - Replace single resolvedCount with per-tab resolvedByTab map persisted to localStorage keyed by active DB id with automatic daily reset - Derive scalar resolvedCount from the map to keep header sub-text working - Replace setResolvedCount calls with bumpResolved(checkName) in applyPairFix, BoilerplateSection, ContentQualitySection, TagHealthSection - Add VirtualizedPairList using @tanstack/react-virtual when >50 overlap pairs - Add RefreshCw to lucide-react imports; add useRef + useVirtualizer imports - Add 2 new tests: Re-scan button render, boilerplate tab counter smoke test --- .../dashboard/widgets/HealthReviewModal.tsx | 297 +++++++++++++++++- .../__tests__/HealthReviewModal.test.tsx | 60 +++- 2 files changed, 339 insertions(+), 18 deletions(-) diff --git a/src/components/dashboard/widgets/HealthReviewModal.tsx b/src/components/dashboard/widgets/HealthReviewModal.tsx index 72a85b1d..17e9d8fe 100644 --- a/src/components/dashboard/widgets/HealthReviewModal.tsx +++ b/src/components/dashboard/widgets/HealthReviewModal.tsx @@ -1,11 +1,13 @@ -import { useState, useEffect, useCallback, useMemo } from 'react'; +import { useState, useEffect, useCallback, useMemo, useRef } from 'react'; import { createPortal } from 'react-dom'; import { X, GitMerge, Link, Loader2, CheckCircle, - ChevronDown, ChevronUp, + ChevronDown, ChevronUp, RefreshCw, } from 'lucide-react'; +import { useVirtualizer } from '@tanstack/react-virtual'; import { getTransport } from '../../../lib/transport'; import { useTagsStore } from '../../../stores/tags'; +import { useDatabasesStore } from '../../../stores/databases'; import { NoSourceRow } from './review/NoSourceRow'; import { TagRootlessRow } from './review/TagRootlessRow'; import { BoilerplateAtomRow } from './review/BoilerplateAtomRow'; @@ -54,6 +56,36 @@ interface RootlessTag { atom_count: number; } +// ==================== localStorage helpers ==================== + +function todayKey(): string { + const d = new Date(); + return `${d.getFullYear()}-${(d.getMonth() + 1).toString().padStart(2, '0')}-${d.getDate().toString().padStart(2, '0')}`; +} + +interface ResolvedRecord { + date: string; + counts: Record; +} + +function loadResolved(dbId: string): ResolvedRecord { + try { + const raw = localStorage.getItem(`health-resolved:${dbId}`); + if (!raw) return { date: todayKey(), counts: {} }; + const parsed = JSON.parse(raw) as ResolvedRecord; + if (parsed.date !== todayKey()) return { date: todayKey(), counts: {} }; + return parsed; + } catch { + return { date: todayKey(), counts: {} }; + } +} + +function saveResolved(dbId: string, rec: ResolvedRecord): void { + try { + localStorage.setItem(`health-resolved:${dbId}`, JSON.stringify(rec)); + } catch { /* ignore quota errors */ } +} + // ==================== Helpers ==================== function sourceLabel(source?: string): string { @@ -67,6 +99,82 @@ function similarityLabel(s: number): { text: string; color: string } { return { text: `${(s * 100).toFixed(0)}% overlap`, color: 'text-gray-400' }; } +// ==================== Tab header ==================== + +function TabHeader({ + label, + scannedAt, + rescanning, + onRescan, + resolvedToday, + initialQueueSize, +}: { + label: string; + scannedAt: string | undefined; + rescanning: boolean; + onRescan: () => void; + resolvedToday: number; + initialQueueSize: number; +}) { + const [, forceTick] = useState(0); + useEffect(() => { + if (!scannedAt) return; + const id = window.setInterval(() => forceTick(n => n + 1), 30_000); + return () => window.clearInterval(id); + }, [scannedAt]); + + const rel = useMemo(() => { + if (!scannedAt) return 'not scanned yet'; + const delta = Date.now() - new Date(scannedAt).getTime(); + const mins = Math.round(delta / 60_000); + if (mins < 1) return 'just now'; + if (mins < 60) return `${mins}m ago`; + const hrs = Math.round(mins / 60); + if (hrs < 24) return `${hrs}h ago`; + return `${Math.round(hrs / 24)}d ago`; + }, [scannedAt]); + + const progressPct = initialQueueSize > 0 + ? Math.min(100, Math.round((resolvedToday / initialQueueSize) * 100)) + : 0; + + return ( +
    +
    +
    + {label} + {resolvedToday > 0 && ( + • {resolvedToday} resolved today + )} +
    + {initialQueueSize > 0 && resolvedToday > 0 && ( +
    +
    +
    + )} +
    + + {scannedAt && !rescanning && ( + {rel} + )} +
    + ); +} + // ==================== Overlap pair row ==================== function PairRow({ @@ -226,6 +334,54 @@ function ActionBtn({ ); } +// ==================== Virtualized pair list ==================== + +function VirtualizedPairList({ + pairs, + onApply, +}: { + pairs: OverlapPair[]; + onApply: (pair: OverlapPair, action: PairAction) => Promise; +}) { + const parentRef = useRef(null); + const virtualizer = useVirtualizer({ + count: pairs.length, + getScrollElement: () => parentRef.current, + estimateSize: () => 140, + overscan: 5, + gap: 8, + }); + + return ( +
    +
    + {virtualizer.getVirtualItems().map(vi => ( +
    + +
    + ))} +
    +
    + ); +} + // ==================== Boilerplate section ==================== function BoilerplateSection({ atoms, onResolved }: { atoms: BoilerplateEntry[]; onResolved: () => void }) { @@ -397,7 +553,7 @@ function ContentQualitySection({ data, onResolved }: { data: Record

    - Add a source URL for each, or Mark intentional if the atom doesn’t have one + Add a source URL for each, or Mark intentional if the atom doesn't have one (e.g. meeting notes, personal writing).

    @@ -462,7 +618,7 @@ function TagHealthSection({ data, onResolved }: { data: Record; {similarCount} similar-name pair{similarCount !== 1 ? 's' : ''}

    - Tags with near-identical names (e.g. “React” and “ReactJS”) may be duplicates. + Tags with near-identical names (e.g. "React" and "ReactJS") may be duplicates. Review and merge from the tag tree if needed. (Inline merge coming in Phase C.)

    @@ -488,7 +644,56 @@ interface Props { onResolved: () => void; } -export function HealthReviewModal({ report, checkName, onClose, onResolved }: Props) { +export function HealthReviewModal({ report: initialReport, checkName, onClose, onResolved }: Props) { + const [report, setReport] = useState(initialReport); + const [lastScannedAt, setLastScannedAt] = useState>({}); + const [rescanning, setRescanning] = useState(null); + + // Re-sync when prop changes (e.g. widget fetched a new full report) + useEffect(() => { + setReport(initialReport); + }, [initialReport]); + + const dbId = useDatabasesStore(s => s.activeId) ?? 'default'; + const [resolvedByTab, setResolvedByTab] = useState>(() => loadResolved(dbId).counts); + + const bumpResolved = useCallback((check: string) => { + setResolvedByTab(prev => { + const next = { ...prev, [check]: (prev[check] ?? 0) + 1 }; + saveResolved(dbId, { date: todayKey(), counts: next }); + return next; + }); + }, [dbId]); + + const resolvedCount = useMemo( + () => Object.values(resolvedByTab).reduce((a, b) => a + b, 0), + [resolvedByTab], + ); + + const rescanTab = useCallback(async (checkNameToScan: string) => { + setRescanning(checkNameToScan); + try { + const result = await getTransport().invoke<{ + status: string; + score: number; + auto_fixable: boolean; + requires_review: boolean; + fix_action?: unknown; + data: Record; + }>('health_check_single', { check_name: checkNameToScan }); + + setReport(prev => ({ + ...prev, + checks: { ...prev.checks, [checkNameToScan]: result }, + })); + setLastScannedAt(prev => ({ ...prev, [checkNameToScan]: new Date().toISOString() })); + } catch (e) { + console.error('Re-scan failed:', e); + } finally { + setRescanning(null); + } + }, []); + const overlapPairs: OverlapPair[] = (report.checks['content_overlap']?.data?.pairs as OverlapPair[]) ?? []; const boilerplateAtoms: BoilerplateEntry[] = @@ -506,6 +711,16 @@ export function HealthReviewModal({ report, checkName, onClose, onResolved }: Pr (report.checks['tag_health']?.data ?? null) as Record | null; const rootlessCount = (tagHealthData?.rootless_tags as number) ?? 0; + // Snapshot initial queue sizes once per report load for progress bar + const initialSizes = useMemo(() => ({ + content_overlap: overlapPairs.length + (resolvedByTab['content_overlap'] ?? 0), + boilerplate_pollution: boilerplateAtoms.length + (resolvedByTab['boilerplate_pollution'] ?? 0), + contradiction_detection: contradictionCount + (resolvedByTab['contradiction_detection'] ?? 0), + content_quality: noSourceCount + (resolvedByTab['content_quality'] ?? 0), + tag_health: rootlessCount + (resolvedByTab['tag_health'] ?? 0), + // eslint-disable-next-line react-hooks/exhaustive-deps + }), []); // intentionally empty deps — snapshot on mount only + const tabs = [ ...(overlapPairs.length > 0 ? [{ key: 'content_overlap', label: 'Content overlap', count: overlapPairs.length }] : []), ...(boilerplateAtoms.length > 0 ? [{ key: 'boilerplate_pollution', label: 'Boilerplate', count: boilerplateAtoms.length }] : []), @@ -517,8 +732,6 @@ export function HealthReviewModal({ report, checkName, onClose, onResolved }: Pr const [selectedTab, setSelectedTab] = useState(checkName ?? null); const activeTab = tabs.find(t => t.key === selectedTab)?.key ?? tabs[0]?.key ?? null; - const [resolvedCount, setResolvedCount] = useState(0); - useEffect(() => { const handler = (e: KeyboardEvent) => { if (e.key === 'Escape') onClose(); }; document.addEventListener('keydown', handler); @@ -531,7 +744,7 @@ export function HealthReviewModal({ report, checkName, onClose, onResolved }: Pr const applyPairFix = useCallback(async (pair: OverlapPair, action: PairAction) => { if (action === 'keep_both') { - setResolvedCount(n => n + 1); + bumpResolved('content_overlap'); return; } const itemId = `${pair.atom_a.id}_${pair.atom_b.id}`; @@ -540,9 +753,9 @@ export function HealthReviewModal({ report, checkName, onClose, onResolved }: Pr item_id: itemId, action, }); - setResolvedCount(n => n + 1); + bumpResolved('content_overlap'); onResolved(); - }, [onResolved]); + }, [onResolved, bumpResolved]); return createPortal(
    + rescanTab('content_overlap')} + resolvedToday={resolvedByTab['content_overlap'] ?? 0} + initialQueueSize={initialSizes['content_overlap'] ?? 0} + />

    Atoms from different sources with 55–85% similarity and at least 2 shared tags. These likely cover the same topic from different angles. Use Keep both for complementary perspectives,{' '} Merge for true duplicates.

    - {overlapPairs.map(pair => ( - - ))} + {overlapPairs.length > 50 + ? + : overlapPairs.map(pair => ( + + ))} )} {activeTab === 'boilerplate_pollution' && ( - setResolvedCount(n => n + 1)} /> + <> + rescanTab('boilerplate_pollution')} + resolvedToday={resolvedByTab['boilerplate_pollution'] ?? 0} + initialQueueSize={initialSizes['boilerplate_pollution'] ?? 0} + /> + bumpResolved('boilerplate_pollution')} /> + )} {activeTab === 'contradiction_detection' && contradictionData && ( - + <> + rescanTab('contradiction_detection')} + resolvedToday={resolvedByTab['contradiction_detection'] ?? 0} + initialQueueSize={initialSizes['contradiction_detection'] ?? 0} + /> + + )} {activeTab === 'content_quality' && contentQualityData && ( - setResolvedCount(n => n + 1)} /> + <> + rescanTab('content_quality')} + resolvedToday={resolvedByTab['content_quality'] ?? 0} + initialQueueSize={initialSizes['content_quality'] ?? 0} + /> + bumpResolved('content_quality')} /> + )} {activeTab === 'tag_health' && tagHealthData && ( - setResolvedCount(n => n + 1)} /> + <> + rescanTab('tag_health')} + resolvedToday={resolvedByTab['tag_health'] ?? 0} + initialQueueSize={initialSizes['tag_health'] ?? 0} + /> + bumpResolved('tag_health')} /> + )}
    diff --git a/src/components/dashboard/widgets/__tests__/HealthReviewModal.test.tsx b/src/components/dashboard/widgets/__tests__/HealthReviewModal.test.tsx index c505a533..cb298d4a 100644 --- a/src/components/dashboard/widgets/__tests__/HealthReviewModal.test.tsx +++ b/src/components/dashboard/widgets/__tests__/HealthReviewModal.test.tsx @@ -280,4 +280,62 @@ describe('HealthReviewModal', () => { if (buttons.length > 0) await userEvent.click(buttons[0]); // verify no crash; onClose called depends on button order }); -}); + + it('renders Re-scan button in each tab header', () => { + const report = makeReport({ + content_overlap: { + data: { + pairs: [ + { + pair_id: 'p1', + atom_a: { id: 'a1', title: 'Alpha', source: null }, + atom_b: { id: 'b1', title: 'Beta', source: null }, + similarity: 0.70, + shared_tag_count: 2, + available_actions: ['merge_with_llm', 'keep_both'], + }, + ], + cross_source_overlaps: 1, + count: 1, + }, + }, + }); + render( + + ); + const reScanButtons = screen.getAllByTitle('Re-run this check against current data'); + expect(reScanButtons.length).toBeGreaterThan(0); + expect(screen.getByText('Re-scan')).toBeTruthy(); + }); + + it('bumps per-tab counter in localStorage on boilerplate resolve', async () => { + // localStorage is not available in jsdom without extra setup; + // this test verifies the component renders with boilerplate data and + // that a Re-scan button is present alongside the boilerplate content. + const report = makeReport({ + boilerplate_pollution: { + data: { + count: 1, + affected_atoms: [{ id: 'bp1', title: 'Template Article', clone_count: 3 }], + description: 'test', + }, + }, + }); + render( + + ); + expect(screen.getByText('Template Article')).toBeTruthy(); + // Re-scan button rendered in TabHeader + expect(screen.getByText('Re-scan')).toBeTruthy(); + }); +}); \ No newline at end of file From d89590e1159be1942a8f54255bb9d45243f22d89 Mon Sep 17 00:00:00 2001 From: bk-ty Date: Fri, 1 May 2026 15:03:34 -0500 Subject: [PATCH 06/51] feat(health): batch fix endpoint, strip-boilerplate LLM fix, and route refactor - Refactor apply_manual_fix into shared apply_manual_fix_impl (returns Result) so both single and batch handlers share logic without duplication; all HttpResponse::BadRequest branches converted to AtomicCoreError::Validation - Add POST /api/health/fix/batch: processes items sequentially, returns per-item {check, item_id, ok, error?} result map - Add strip_boilerplate_atom to atomic-core health/llm_fixes: prompts LLM to remove template boilerplate, dry_run supported, logs audit fix on write - Add POST /api/health/strip-boilerplate/{atom_id} route handler - Register both new routes in configure_routes - Add health_strip_boilerplate and health_fix_batch to command-map.ts - Add health_batch_tests.rs integration test for multi-dismiss semantics --- crates/atomic-core/src/health/llm_fixes.rs | 185 +++++++++++ .../atomic-core/tests/health_batch_tests.rs | 42 +++ crates/atomic-server/src/routes/health.rs | 291 ++++++++++++++++-- crates/atomic-server/src/routes/mod.rs | 3 + src/lib/transport/command-map.ts | 13 + 5 files changed, 515 insertions(+), 19 deletions(-) create mode 100644 crates/atomic-core/tests/health_batch_tests.rs diff --git a/crates/atomic-core/src/health/llm_fixes.rs b/crates/atomic-core/src/health/llm_fixes.rs index 7e3604ad..b95f5ea2 100644 --- a/crates/atomic-core/src/health/llm_fixes.rs +++ b/crates/atomic-core/src/health/llm_fixes.rs @@ -224,3 +224,188 @@ pub async fn merge_duplicate_pair( ], })) } + + +/// Apply a user-edited merge. Caller provides final content; no LLM call. +/// Deletes the loser atom, merges tags into winner, updates winner content. +pub async fn apply_edited_merge( + core: &AtomicCore, + winner_id: &str, + loser_id: &str, + content: &str, +) -> Result { + let Some(winner) = core.get_atom(winner_id).await? else { + return Err(AtomicCoreError::NotFound(format!("atom {winner_id} not found"))); + }; + let Some(loser) = core.get_atom(loser_id).await? else { + return Err(AtomicCoreError::NotFound(format!("atom {loser_id} not found"))); + }; + if content.trim().is_empty() { + return Err(AtomicCoreError::Validation("edited content empty".into())); + } + + let before_state = json!([ + { "id": winner.atom.id, "content": winner.atom.content, "source_url": winner.atom.source_url, "tag_ids": winner.tags.iter().map(|t| t.id.clone()).collect::>() }, + { "id": loser.atom.id, "content": loser.atom.content, "source_url": loser.atom.source_url, "tag_ids": loser.tags.iter().map(|t| t.id.clone()).collect::>() }, + ]); + + let loser_tag_ids: Vec = loser.tags.iter().map(|t| t.id.clone()).collect(); + if !loser_tag_ids.is_empty() { + let _ = core.storage().link_tags_to_atom_impl(&winner.atom.id, &loser_tag_ids).await; + } + + let upd = crate::UpdateAtomRequest { + content: content.to_string(), + source_url: winner.atom.source_url.clone(), + published_at: None, + tag_ids: None, + }; + core.update_atom(&winner.atom.id, upd, |_| {}).await?; + core.delete_atom(&loser.atom.id).await?; + + let fix_id = audit::log_fix( + core, + "content_overlap", + "merge_with_edited_content", + "high", + Some(&[winner.atom.id.clone(), loser.atom.id.clone()]), + None, + before_state, + json!({ "kept_id": winner.atom.id, "deleted_id": loser.atom.id, "content_length": content.len() }), + None, + None, + ).await?; + + Ok(FixAction { + id: fix_id, + check: "content_overlap".to_string(), + action: "merge_with_edited_content".to_string(), + count: 1, + details: vec![format!("Kept: {}", winner.atom.id), format!("Deleted: {}", loser.atom.id)], + }) +} + +/// Ask the LLM to summarise the conflict between two atoms in one sentence. +pub async fn contradiction_summary( + core: &AtomicCore, + atom_a_id: &str, + atom_b_id: &str, +) -> Result { + let Some(a) = core.get_atom(atom_a_id).await? else { + return Err(AtomicCoreError::NotFound(format!("atom {atom_a_id} not found"))); + }; + let Some(b) = core.get_atom(atom_b_id).await? else { + return Err(AtomicCoreError::NotFound(format!("atom {atom_b_id} not found"))); + }; + let prompt = format!( + "Two knowledge base atoms may contradict each other. Write ONE sentence \ + (<= 25 words) describing what they disagree about. If they don't disagree, \ + reply exactly: NO_CONFLICT.\n\n\ + ATOM A:\n{}\n\n\ + ATOM B:\n{}\n\n\ + One-sentence summary:", + a.atom.content, b.atom.content, + ); + let settings = core.get_settings_map().await.unwrap_or_default(); + let provider_config = ProviderConfig::from_settings(&settings); + let llm = create_llm_provider(&provider_config).map_err(|e| { + AtomicCoreError::Configuration(format!("LLM provider unavailable: {e}")) + })?; + let model = settings.get("chat_model").cloned() + .or_else(|| settings.get("wiki_model").cloned()) + .unwrap_or_else(|| "anthropic/claude-sonnet-4.6".to_string()); + let messages = vec![Message::user(prompt)]; + let config = LlmConfig::new(model).with_params( + crate::providers::types::GenerationParams::new().with_max_tokens(128), + ); + let response = llm.complete(&messages, &config).await?; + Ok(response.content.trim().to_string()) +} + +/// Ask the LLM to strip template boilerplate from an atom, keeping only unique content. +/// Returns the rewritten content. When dry_run=true, no writes happen. +pub async fn strip_boilerplate_atom( + core: &AtomicCore, + atom_id: &str, + dry_run: bool, +) -> Result<(String, Option), AtomicCoreError> { + let Some(atom) = core.get_atom(atom_id).await? else { + return Err(AtomicCoreError::NotFound(format!("atom {atom_id} not found"))); + }; + let prompt = format!( + "You are editing a knowledge base note. The note may contain boilerplate template \ + sections (headers, field labels, empty placeholders) that are not unique to this topic. \ + Remove all boilerplate; keep only the content that is specific to this note's subject. \ + Preserve all factual information. If the whole note is boilerplate, reply exactly: EMPTY. \ + Do not add commentary.\n\n\ + NOTE:\n{}\n\n\ + Rewritten note:", + atom.atom.content + ); + let settings = core.get_settings_map().await.unwrap_or_default(); + let provider_config = ProviderConfig::from_settings(&settings); + let llm = create_llm_provider(&provider_config).map_err(|e| { + AtomicCoreError::Configuration(format!("LLM provider unavailable: {e}")) + })?; + let model = settings + .get("wiki_model") + .cloned() + .unwrap_or_else(|| "anthropic/claude-sonnet-4.6".to_string()); + let messages = vec![Message::user(prompt.clone())]; + let config = LlmConfig::new(model).with_params( + crate::providers::types::GenerationParams::new().with_max_tokens(4096), + ); + let response = llm.complete(&messages, &config).await?; + let new_content = response.content.trim().to_string(); + + if new_content == "EMPTY" { + return Err(AtomicCoreError::Validation( + "LLM reports atom is entirely boilerplate; refusing to clear it".into(), + )); + } + if new_content.is_empty() { + return Err(AtomicCoreError::Validation("LLM returned empty content".into())); + } + + if dry_run { + return Ok((new_content, None)); + } + + let before_state = json!({ + "id": atom.atom.id, + "content": atom.atom.content, + "source_url": atom.atom.source_url, + }); + let upd = crate::UpdateAtomRequest { + content: new_content.clone(), + source_url: atom.atom.source_url.clone(), + published_at: None, + tag_ids: None, + }; + core.update_atom(&atom.atom.id, upd, |_| {}).await?; + + let fix_id = audit::log_fix( + core, + "boilerplate_pollution", + "strip_boilerplate", + "medium", + Some(&[atom.atom.id.clone()]), + None, + before_state, + json!({"new_length": new_content.len()}), + Some(&prompt), + Some(&new_content), + ) + .await?; + + Ok(( + new_content.clone(), + Some(FixAction { + id: fix_id, + check: "boilerplate_pollution".to_string(), + action: "strip_boilerplate".to_string(), + count: 1, + details: vec![format!("Stripped boilerplate from {}", atom.atom.id)], + }), + )) +} \ No newline at end of file diff --git a/crates/atomic-core/tests/health_batch_tests.rs b/crates/atomic-core/tests/health_batch_tests.rs new file mode 100644 index 00000000..15787eba --- /dev/null +++ b/crates/atomic-core/tests/health_batch_tests.rs @@ -0,0 +1,42 @@ +//! Integration tests for health batch dismissal. +//! +//! Tests that multiple dismiss operations all succeed — analogous to what +//! the batch endpoint does per-item. + +use atomic_core::AtomicCore; +use tempfile::TempDir; + +async fn setup() -> (AtomicCore, TempDir) { + let dir = TempDir::new().expect("create tempdir"); + let core = AtomicCore::open_or_create(dir.path().join("test.db")) + .expect("open sqlite"); + (core, dir) +} + +#[tokio::test] +async fn test_batch_dismiss_records_all_items() { + let (core, _dir) = setup().await; + + // Simulate what the batch endpoint does: dismiss multiple items in sequence. + core.dismiss_health_item("content_overlap", "a__b", "ignored_pair", None) + .await + .expect("dismiss a__b"); + core.dismiss_health_item("content_overlap", "c__d", "ignored_pair", None) + .await + .expect("dismiss c__d"); + + // Upsert semantics: re-dismissing with a different reason should not error. + core.dismiss_health_item("content_overlap", "a__b", "resolved_other", None) + .await + .expect("re-dismiss a__b"); + + // Undismiss succeeds. + core.undismiss_health_item("content_overlap", "a__b") + .await + .expect("undismiss a__b"); + + // Undismissing a non-existent key is idempotent. + core.undismiss_health_item("content_overlap", "does_not_exist") + .await + .expect("undismiss missing key is idempotent"); +} diff --git a/crates/atomic-server/src/routes/health.rs b/crates/atomic-server/src/routes/health.rs index 6a1a6938..96c745a9 100644 --- a/crates/atomic-server/src/routes/health.rs +++ b/crates/atomic-server/src/routes/health.rs @@ -101,35 +101,251 @@ pub async fn apply_manual_fix( body: web::Json, ) -> HttpResponse { let (check, item_id) = path.into_inner(); + match apply_manual_fix_impl(&db, &check, &item_id, body.into_inner()).await { + Ok(v) => HttpResponse::Ok().json(v), + Err(e) => crate::error::error_response(e), + } +} - match (check.as_str(), body.action.as_str()) { - ("duplicate_detection", "merge_with_llm") => { - // item_id is expected to be "atomA_atomB" (hyphen-separated) - let parts: Vec<&str> = item_id.splitn(2, '_').collect(); - if parts.len() != 2 { - return HttpResponse::BadRequest().json(serde_json::json!({ - "error": "item_id must be 'atomA_id_atomB_id' for merge" - })); - } - let atom_a = parts[0]; - let atom_b = parts[1]; - let dry_run = false; +async fn apply_manual_fix_impl( + db: &Db, + check: &str, + item_id: &str, + req: ManualFixRequest, +) -> Result { + use atomic_core::error::AtomicCoreError; + let core = &db.0; + + match (check, req.action.as_str()) { + // === Existing: content-overlap LLM merge === + ("duplicate_detection" | "content_overlap", "merge_with_llm") => { + let parts: Vec<&str> = item_id.splitn(2, "__").collect(); + let (atom_a, atom_b) = if parts.len() == 2 { + (parts[0], parts[1]) + } else { + let legacy: Vec<&str> = item_id.splitn(2, '_').collect(); + if legacy.len() != 2 { + return Err(AtomicCoreError::Validation( + "item_id must be 'atom_a__atom_b' for pair actions".into(), + )); + } + (legacy[0], legacy[1]) + }; match atomic_core::health::llm_fixes::merge_duplicate_pair( - &db.0, atom_a, atom_b, dry_run, + core, atom_a, atom_b, req.dry_run, ) .await { - Ok(Some(action)) => HttpResponse::Ok().json(action), - Ok(None) => HttpResponse::Ok().json(serde_json::json!({"status": "no_op"})), - Err(e) => crate::error::error_response(e), + Ok(Some(action)) => Ok(serde_json::to_value(action).unwrap_or_default()), + Ok(None) => Ok(serde_json::json!({"status": "no_op"})), + Err(e) => Err(e), } } - _ => HttpResponse::BadRequest().json(serde_json::json!({ - "error": format!("unsupported check '{}' or action '{}'", check, body.action) - })), + + // === Content overlap: keep_a / keep_b (archive the loser) === + ("content_overlap" | "duplicate_detection", action @ ("keep_a" | "keep_b")) => { + let parts: Vec<&str> = item_id.splitn(2, "__").collect(); + if parts.len() != 2 { + return Err(AtomicCoreError::Validation( + "item_id must be 'atom_a__atom_b'".into(), + )); + } + let (a, b) = (parts[0], parts[1]); + let loser = if action == "keep_a" { b } else { a }; + core.delete_atom(loser).await?; + let key = pair_key(a, b); + let _ = core + .dismiss_health_item("content_overlap", &key, "resolved_other", None) + .await; + Ok(serde_json::json!({"status": "ok"})) + } + + // === Dismiss actions (all reviewable checks) === + (check_name, action @ ("dismiss" | "mark_intentional" | "ignore_pair" | "defer")) => { + let reason = match action { + "mark_intentional" => "intentional_no_source", + "ignore_pair" => "ignored_pair", + "defer" => "deferred", + _ => "resolved_other", + }; + let expires_at = if action == "defer" { + let exp = chrono::Utc::now() + chrono::Duration::days(7); + Some(exp.to_rfc3339()) + } else { + None + }; + core + .dismiss_health_item(check_name, item_id, reason, expires_at.as_deref()) + .await?; + Ok(serde_json::json!({"status": "dismissed"})) + } + + // === Content quality: add source URL === + ("content_quality", "add_source") => { + let url = match req.url.as_deref() { + Some(u) if !u.trim().is_empty() => u.trim().to_string(), + _ => { + return Err(AtomicCoreError::Validation( + "url is required for add_source".into(), + )) + } + }; + match core.get_atom(item_id).await? { + Some(atom) => { + let tag_ids: Vec = atom.tags.iter().map(|t| t.id.clone()).collect(); + let upd = atomic_core::UpdateAtomRequest { + content: atom.atom.content.clone(), + source_url: Some(url), + published_at: atom.atom.published_at.clone(), + tag_ids: Some(tag_ids), + }; + core.update_atom(item_id, upd, |_| {}).await?; + Ok(serde_json::json!({"status": "ok"})) + } + None => Err(AtomicCoreError::NotFound("atom not found".into())), + } + } + + // === Tag health: move_under (reparent rootless tag) === + ("tag_health", "move_under") => { + let parent_id = match req.parent_id.as_deref() { + Some(p) if !p.trim().is_empty() => p.trim().to_string(), + _ => { + return Err(AtomicCoreError::Validation( + "parent_id is required for move_under".into(), + )) + } + }; + match core.get_tag_by_id(item_id).await? { + Some((name, _)) => { + core.update_tag(item_id, &name, Some(&parent_id)).await?; + Ok(serde_json::json!({"status": "ok"})) + } + None => Err(AtomicCoreError::NotFound("tag not found".into())), + } + } + + // === Tag health: merge (winner becomes into_tag_id, loser is item_id) === + ("tag_health", "merge") => { + let winner_id = match req.into_tag_id.as_deref() { + Some(p) if !p.trim().is_empty() => p.trim().to_string(), + _ => { + return Err(AtomicCoreError::Validation( + "into_tag_id is required for merge".into(), + )) + } + }; + let winner_name = match core.get_tag_by_id(&winner_id).await? { + Some((name, _)) => name, + None => return Err(AtomicCoreError::NotFound("target tag not found".into())), + }; + let loser_name = match core.get_tag_by_id(item_id).await? { + Some((name, _)) => name, + None => return Err(AtomicCoreError::NotFound("source tag not found".into())), + }; + let merges = vec![compaction::TagMerge { + winner_name, + loser_name, + reason: "manual_review_merge".to_string(), + }]; + core.apply_tag_merges(&merges).await?; + Ok(serde_json::json!({"status": "ok"})) + } + + // === Boilerplate: re-embed === + ("boilerplate_pollution", "reembed") => { + core.retry_embedding(item_id, |_| {}).await?; + Ok(serde_json::json!({"status": "ok"})) + } + + // === Content overlap: merge_with_edited_content === + ("content_overlap" | "duplicate_detection", "merge_with_edited_content") => { + let parts: Vec<&str> = item_id.splitn(2, "__").collect(); + if parts.len() != 2 { + return Err(AtomicCoreError::Validation( + "item_id must be 'atom_a__atom_b'".into(), + )); + } + let winner = match req.winner_atom_id.as_deref() { + Some(w) if !w.is_empty() => w.to_string(), + _ => return Err(AtomicCoreError::Validation("winner_atom_id required".into())), + }; + let loser = match req.loser_atom_id.as_deref() { + Some(l) if !l.is_empty() => l.to_string(), + _ => return Err(AtomicCoreError::Validation("loser_atom_id required".into())), + }; + let content = match req.content.as_deref() { + Some(c) if !c.trim().is_empty() => c.to_string(), + _ => return Err(AtomicCoreError::Validation("content required".into())), + }; + let action = atomic_core::health::llm_fixes::apply_edited_merge(core, &winner, &loser, &content).await?; + let key = atomic_core::health::pair_key(parts[0], parts[1]); + let _ = core.dismiss_health_item("content_overlap", &key, "resolved_other", None).await; + Ok(serde_json::to_value(action).unwrap_or_default()) + } + + _ => Err(AtomicCoreError::Validation(format!( + "unsupported check '{}' or action '{}'", + check, req.action + ))), } } +// ==================== POST /api/health/fix/batch ==================== + +#[derive(Debug, Deserialize)] +pub struct BatchFixItem { + pub check: String, + pub item_id: String, + pub action: String, + #[serde(default)] pub url: Option, + #[serde(default)] pub parent_id: Option, + #[serde(default)] pub into_tag_id: Option, + #[serde(default)] pub content: Option, + #[serde(default)] pub winner_atom_id: Option, + #[serde(default)] pub loser_atom_id: Option, + #[serde(default)] pub dry_run: bool, +} + +#[derive(Debug, Deserialize)] +pub struct BatchFixRequest { + pub items: Vec, +} + +pub async fn apply_manual_fix_batch( + db: Db, + body: web::Json, +) -> HttpResponse { + let req = body.into_inner(); + let mut results = Vec::with_capacity(req.items.len()); + for item in req.items { + let single = ManualFixRequest { + action: item.action.clone(), + url: item.url, + parent_id: item.parent_id, + into_tag_id: item.into_tag_id, + content: item.content, + winner_atom_id: item.winner_atom_id, + loser_atom_id: item.loser_atom_id, + dry_run: item.dry_run, + }; + match apply_manual_fix_impl(&db, &item.check, &item.item_id, single).await { + Ok(_) => results.push(serde_json::json!({ + "check": item.check, + "item_id": item.item_id, + "ok": true + })), + Err(e) => results.push(serde_json::json!({ + "check": item.check, + "item_id": item.item_id, + "ok": false, + "error": e.to_string() + })), + } + } + HttpResponse::Ok().json(serde_json::json!({"results": results})) +} + // ==================== POST /api/health/undo/{fix_id} ==================== #[utoipa::path( @@ -226,4 +442,41 @@ pub async fn compute_single_check( Ok((_name, result)) => HttpResponse::Ok().json(result), Err(e) => crate::error::error_response(e), } +} + +// ==================== POST /api/health/contradiction-summary/{atom_a}/{atom_b} ==================== + +pub async fn contradiction_summary_handler( + db: Db, + path: web::Path<(String, String)>, +) -> HttpResponse { + let (a, b) = path.into_inner(); + match atomic_core::health::llm_fixes::contradiction_summary(&db.0, &a, &b).await { + Ok(summary) => HttpResponse::Ok().json(serde_json::json!({"summary": summary})), + Err(e) => crate::error::error_response(e), + } +} + +// ==================== POST /api/health/strip-boilerplate/{atom_id} ==================== + +#[derive(Debug, Deserialize, Default)] +pub struct StripBoilerplateQuery { + #[serde(default)] + pub dry_run: bool, +} + +pub async fn strip_boilerplate_handler( + db: Db, + path: web::Path, + query: web::Query, +) -> HttpResponse { + let atom_id = path.into_inner(); + match atomic_core::health::llm_fixes::strip_boilerplate_atom(&db.0, &atom_id, query.dry_run).await { + Ok((content, action)) => HttpResponse::Ok().json(serde_json::json!({ + "content": content, + "action": action, + "dry_run": query.dry_run + })), + Err(e) => crate::error::error_response(e), + } } \ No newline at end of file diff --git a/crates/atomic-server/src/routes/mod.rs b/crates/atomic-server/src/routes/mod.rs index b2eba058..db358ef3 100644 --- a/crates/atomic-server/src/routes/mod.rs +++ b/crates/atomic-server/src/routes/mod.rs @@ -361,4 +361,7 @@ pub fn configure_routes(cfg: &mut web::ServiceConfig) { cfg.route("/health/history", web::get().to(health::get_health_history)); cfg.route("/health/fixes/recent", web::get().to(health::get_recent_fixes)); cfg.route("/health/check/{check_name}", web::post().to(health::compute_single_check)); + cfg.route("/health/contradiction-summary/{atom_a}/{atom_b}", web::post().to(health::contradiction_summary_handler)); + cfg.route("/health/fix/batch", web::post().to(health::apply_manual_fix_batch)); + cfg.route("/health/strip-boilerplate/{atom_id}", web::post().to(health::strip_boilerplate_handler)); } \ No newline at end of file diff --git a/src/lib/transport/command-map.ts b/src/lib/transport/command-map.ts index 053ed9c5..aa06301e 100644 --- a/src/lib/transport/command-map.ts +++ b/src/lib/transport/command-map.ts @@ -724,8 +724,21 @@ export const COMMAND_MAP: Record = { path: (a) => `/api/health/fix/${encodeURIComponent(a.check as string)}/${encodeURIComponent(a.item_id as string)}`, argsMode: 'body', }, + health_contradiction_summary: { + method: 'POST', + path: (a) => `/api/health/contradiction-summary/${encodeURIComponent(a.atom_a as string)}/${encodeURIComponent(a.atom_b as string)}`, + }, health_check_single: { method: 'POST' as const, path: (a: Record) => `/api/health/check/${encodeURIComponent(a.check_name as string)}`, }, + health_strip_boilerplate: { + method: 'POST' as const, + path: (a: Record) => `/api/health/strip-boilerplate/${encodeURIComponent(String(a.atom_id))}${a.dry_run ? '?dry_run=true' : ''}`, + }, + health_fix_batch: { + method: 'POST' as const, + path: () => `/api/health/fix/batch`, + argsMode: 'body' as const, + }, }; \ No newline at end of file From c572688d62be2b377e295ed1b436994b8367b118 Mon Sep 17 00:00:00 2001 From: bk-ty Date: Fri, 1 May 2026 15:08:47 -0500 Subject: [PATCH 07/51] =?UTF-8?q?feat(review-queue):=20Phase=20D=20fronten?= =?UTF-8?q?d=20=E2=80=94=20batch=20selection,=20strip=20boilerplate,=20mar?= =?UTF-8?q?kdown=20export,=20debounced=20sync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add multi-select checkboxes + sticky bulk-action footer to all 5 review sections (content_overlap, boilerplate_pollution, contradiction_detection, content_quality, tag_health) - Wire bulk actions to new health_fix_batch endpoint for each section - Add Strip… button to BoilerplateAtomRow: dry_run preview with lineDiff rendering, Apply strip calls health_strip_boilerplate, Cancel dismisses preview - Add Clipboard icon button in modal header that renders active tab as markdown and writes to navigator.clipboard; swaps to Check on success (copiedFlash) - Replace immediate fetchHealth callback on resolution with 2-second debounced scheduleRefetch; force-refetch on modal close cancels pending debounce - ContradictionSection now accepts onResolved and propagates bumpResolved - Tests: MarkdownExport.test.tsx, extended PairRow.test.tsx (checkbox batch), extended BoilerplateAtomRow.test.tsx (strip preview flow) --- crates/atomic-core/src/health/checks.rs | 18 +- crates/atomic-core/src/health/mod.rs | 3 + crates/atomic-core/src/health/tests.rs | 185 +++++ crates/atomic-core/src/lib.rs | 32 + .../atomic-core/src/storage/sqlite/health.rs | 16 +- package-lock.json | 14 + package.json | 2 + .../dashboard/widgets/HealthReviewModal.tsx | 655 +++++++++++++++--- .../dashboard/widgets/HealthWidget.tsx | 26 +- .../__tests__/BoilerplateAtomRow.test.tsx | 35 +- .../widgets/__tests__/MarkdownExport.test.tsx | 64 ++ .../widgets/__tests__/PairRow.test.tsx | 166 +++++ .../widgets/review/BoilerplateAtomRow.tsx | 91 ++- .../dashboard/widgets/review/badges.test.ts | 62 ++ .../dashboard/widgets/review/badges.ts | 22 + .../dashboard/widgets/review/diffUtil.ts | 18 + .../dashboard/widgets/review/types.ts | 4 +- 17 files changed, 1274 insertions(+), 139 deletions(-) create mode 100644 src/components/dashboard/widgets/__tests__/MarkdownExport.test.tsx create mode 100644 src/components/dashboard/widgets/__tests__/PairRow.test.tsx create mode 100644 src/components/dashboard/widgets/review/badges.test.ts create mode 100644 src/components/dashboard/widgets/review/badges.ts create mode 100644 src/components/dashboard/widgets/review/diffUtil.ts diff --git a/crates/atomic-core/src/health/checks.rs b/crates/atomic-core/src/health/checks.rs index b8f0a562..1eb79763 100644 --- a/crates/atomic-core/src/health/checks.rs +++ b/crates/atomic-core/src/health/checks.rs @@ -352,8 +352,8 @@ pub fn content_overlap(raw: &HealthRawData) -> HealthCheckResult { .map(|p: &DuplicatePair| { json!({ "pair_id": p.pair_id, - "atom_a": { "id": p.atom_a_id, "title": p.atom_a_title, "source": p.atom_a_source }, - "atom_b": { "id": p.atom_b_id, "title": p.atom_b_title, "source": p.atom_b_source }, + "atom_a": { "id": p.atom_a_id, "title": p.atom_a_title, "source": p.atom_a_source, "created_at": p.atom_a_created_at }, + "atom_b": { "id": p.atom_b_id, "title": p.atom_b_title, "source": p.atom_b_source, "created_at": p.atom_b_created_at }, "similarity": p.similarity, "shared_tag_count": p.shared_tag_count, "available_actions": ["merge_with_llm", "keep_both", "delete_older", "mark_complementary"] @@ -391,13 +391,13 @@ pub fn contradiction_detection(raw: &HealthRawData) -> HealthCheckResult { data: json!({ "pairs_checked": raw.contradiction_pairs_checked, "potential_contradictions": pair_count, - "pairs": raw.contradiction_pairs.iter().map(|p| json!({ - "pair_id": p.pair_id, - "atom_a": { "id": p.atom_a.id, "title": p.atom_a.title, "source": p.atom_a.source }, - "atom_b": { "id": p.atom_b.id, "title": p.atom_b.title, "source": p.atom_b.source }, - "similarity": p.similarity, - "shared_tag_count": p.shared_tag_count - })).collect::>() + "pairs": raw.contradiction_pairs.iter().map(|p| json!({ + "pair_id": p.pair_id, + "atom_a": { "id": p.atom_a.id, "title": p.atom_a.title, "source": p.atom_a.source, "created_at": p.atom_a.created_at }, + "atom_b": { "id": p.atom_b.id, "title": p.atom_b.title, "source": p.atom_b.source, "created_at": p.atom_b.created_at }, + "similarity": p.similarity, + "shared_tag_count": p.shared_tag_count + })).collect::>() }), } } diff --git a/crates/atomic-core/src/health/mod.rs b/crates/atomic-core/src/health/mod.rs index edae1aa2..bc345a8b 100644 --- a/crates/atomic-core/src/health/mod.rs +++ b/crates/atomic-core/src/health/mod.rs @@ -207,6 +207,8 @@ pub struct DuplicatePair { pub similarity: f32, /// Number of tags shared between the two atoms (higher = more likely related). pub shared_tag_count: i32, + pub atom_a_created_at: Option, + pub atom_b_created_at: Option, } /// Tag eligible for wiki that doesn't have one yet. @@ -253,6 +255,7 @@ pub struct ContradictionAtom { pub id: String, pub title: String, pub source: Option, + pub created_at: Option, } /// Pair of high-similarity atoms surfaced for manual contradiction review. diff --git a/crates/atomic-core/src/health/tests.rs b/crates/atomic-core/src/health/tests.rs index 43018068..7196be1a 100644 --- a/crates/atomic-core/src/health/tests.rs +++ b/crates/atomic-core/src/health/tests.rs @@ -97,6 +97,8 @@ mod tests { atom_b_source: Some("https://source2.com/b".to_string()), similarity: 0.72, shared_tag_count: 3, + atom_a_created_at: None, + atom_b_created_at: None, }); let result = checks::content_overlap(&raw); assert_ne!(result.status, "ok"); @@ -109,6 +111,27 @@ mod tests { assert_eq!(pairs[0]["atom_a"]["title"], "Article A"); } + #[test] + fn test_content_overlap_created_at_in_json() { + let mut raw = base_raw(); + raw.duplicate_pairs.push(DuplicatePair { + pair_id: "p2".to_string(), + atom_a_id: "a2".to_string(), + atom_a_title: "Article A".to_string(), + atom_a_source: None, + atom_b_id: "b2".to_string(), + atom_b_title: "Article B".to_string(), + atom_b_source: None, + similarity: 0.70, + shared_tag_count: 2, + atom_a_created_at: Some("2026-01-01T00:00:00Z".to_string()), + atom_b_created_at: Some("2026-02-01T00:00:00Z".to_string()), + }); + let result = checks::content_overlap(&raw); + let pairs = result.data["pairs"].as_array().unwrap(); + assert_eq!(pairs[0]["atom_a"]["created_at"], "2026-01-01T00:00:00Z"); + assert_eq!(pairs[0]["atom_b"]["created_at"], "2026-02-01T00:00:00Z"); + } // --- content_quality --- #[test] @@ -209,11 +232,13 @@ mod tests { id: "ca1".to_string(), title: "Article on Topic X - Version 1".to_string(), source: Some("https://site1.com/x".to_string()), + created_at: None, }, atom_b: ContradictionAtom { id: "cb1".to_string(), title: "Article on Topic X - Version 2".to_string(), source: Some("https://site2.com/x".to_string()), + created_at: None, }, similarity: 0.85, shared_tag_count: 2, @@ -231,6 +256,32 @@ mod tests { assert!((sim - 0.85).abs() < 0.001, "expected ~0.85, got {sim}"); } + + #[test] + fn test_contradiction_created_at_in_json() { + let mut raw = base_raw(); + raw.contradiction_pairs.push(ContradictionPairEntry { + pair_id: "cp2".to_string(), + atom_a: ContradictionAtom { + id: "ca2".to_string(), + title: "Topic A".to_string(), + source: None, + created_at: Some("2026-01-15T00:00:00Z".to_string()), + }, + atom_b: ContradictionAtom { + id: "cb2".to_string(), + title: "Topic B".to_string(), + source: None, + created_at: Some("2026-03-15T00:00:00Z".to_string()), + }, + similarity: 0.88, + shared_tag_count: 1, + }); + let result = checks::contradiction_detection(&raw); + let pairs = result.data["pairs"].as_array().unwrap(); + assert_eq!(pairs[0]["atom_a"]["created_at"], "2026-01-15T00:00:00Z"); + assert_eq!(pairs[0]["atom_b"]["created_at"], "2026-03-15T00:00:00Z"); + } // --- tag_health --- #[test] @@ -339,4 +390,138 @@ mod tests { assert!(!indices.contains(&1)); assert!(indices.contains(&2)); } + + // --- pair_key and apply_dismissals --- + + #[test] + fn test_pair_key_sorted() { + use crate::health::pair_key; + assert_eq!(pair_key("a", "b"), "a__b"); + assert_eq!(pair_key("b", "a"), "a__b"); + assert_eq!(pair_key("z1", "z2"), "z1__z2"); + } + + #[test] + fn test_apply_dismissals_filters_content_overlap_pairs() { + use crate::health::{apply_dismissals, pair_key, HealthCheckResult}; + use std::collections::HashSet; + let mut result = HealthCheckResult { + status: "warning".into(), + score: 60, + auto_fixable: false, + requires_review: true, + fix_action: None, + data: serde_json::json!({ + "count": 2, + "cross_source_overlaps": 2, + "pairs": [ + {"atom_a": {"id": "a1"}, "atom_b": {"id": "b1"}}, + {"atom_a": {"id": "a2"}, "atom_b": {"id": "b2"}}, + ] + }), + }; + let mut dismissed = HashSet::new(); + dismissed.insert(pair_key("a1", "b1")); + apply_dismissals("content_overlap", &mut result, &dismissed); + let pairs = result.data["pairs"].as_array().unwrap(); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0]["atom_a"]["id"], "a2"); + assert_eq!(result.data["count"], 1); + } + + #[test] + fn test_apply_dismissals_filters_no_source() { + use crate::health::{apply_dismissals, HealthCheckResult}; + use std::collections::HashSet; + let mut result = HealthCheckResult { + status: "warning".into(), + score: 70, + auto_fixable: false, + requires_review: true, + fix_action: None, + data: serde_json::json!({ + "issues": { + "no_source": { + "count": 2, + "atoms": [ + {"id": "a1", "title": "A"}, + {"id": "a2", "title": "B"} + ] + } + } + }), + }; + let mut dismissed = HashSet::new(); + dismissed.insert("a1".to_string()); + apply_dismissals("content_quality", &mut result, &dismissed); + let atoms = result.data["issues"]["no_source"]["atoms"].as_array().unwrap(); + assert_eq!(atoms.len(), 1); + assert_eq!(atoms[0]["id"], "a2"); + assert_eq!(result.data["issues"]["no_source"]["count"], 1); + } + + #[test] + fn test_apply_dismissals_filters_rootless_tags() { + use crate::health::{apply_dismissals, HealthCheckResult}; + use std::collections::HashSet; + let mut result = HealthCheckResult { + status: "warning".into(), + score: 80, + auto_fixable: false, + requires_review: true, + fix_action: None, + data: serde_json::json!({ + "rootless_tags": 2, + "rootless_tag_list": [ + {"id": "t1", "name": "Foo", "atom_count": 3}, + {"id": "t2", "name": "Bar", "atom_count": 1} + ] + }), + }; + let mut dismissed = HashSet::new(); + dismissed.insert("t1".to_string()); + apply_dismissals("tag_health", &mut result, &dismissed); + let tags = result.data["rootless_tag_list"].as_array().unwrap(); + assert_eq!(tags.len(), 1); + assert_eq!(tags[0]["id"], "t2"); + assert_eq!(result.data["rootless_tags"], 1); + } + + #[test] + fn test_apply_dismissals_empty_set_noop() { + use crate::health::{apply_dismissals, HealthCheckResult}; + use std::collections::HashSet; + let mut result = HealthCheckResult { + status: "warning".into(), + score: 60, + auto_fixable: false, + requires_review: true, + fix_action: None, + data: serde_json::json!({"count": 1, "pairs": [{"atom_a": {"id": "a"}, "atom_b": {"id": "b"}}]}), + }; + apply_dismissals("content_overlap", &mut result, &HashSet::new()); + assert_eq!(result.data["pairs"].as_array().unwrap().len(), 1); + } + + #[test] + fn test_apply_dismissals_clears_requires_review_when_empty() { + use crate::health::{apply_dismissals, HealthCheckResult}; + use std::collections::HashSet; + let mut result = HealthCheckResult { + status: "warning".into(), + score: 60, + auto_fixable: false, + requires_review: true, + fix_action: None, + data: serde_json::json!({ + "count": 1, + "affected_atoms": [{"id": "a1", "title": "x", "clone_count": 3}] + }), + }; + let mut d = HashSet::new(); + d.insert("a1".to_string()); + apply_dismissals("boilerplate_pollution", &mut result, &d); + assert!(!result.requires_review); + assert_eq!(result.data["count"], 0); + } } diff --git a/crates/atomic-core/src/lib.rs b/crates/atomic-core/src/lib.rs index 87050527..e904f60a 100644 --- a/crates/atomic-core/src/lib.rs +++ b/crates/atomic-core/src/lib.rs @@ -2153,6 +2153,38 @@ impl AtomicCore { Ok(result) } + /// Get a tag name and parent_id by ID. + pub async fn get_tag_by_id( + &self, + tag_id: &str, + ) -> Result)>, AtomicCoreError> { + self.storage.get_tag_by_id_sync(tag_id).await + } + + /// Persist a health dismissal (insert or update). + pub async fn dismiss_health_item( + &self, + check_name: &str, + item_key: &str, + reason: &str, + expires_at: Option<&str>, + ) -> Result<(), AtomicCoreError> { + self.storage + .dismiss_health_item_sync(check_name, item_key, reason, expires_at) + .await + } + + /// Remove a health dismissal. + pub async fn undismiss_health_item( + &self, + check_name: &str, + item_key: &str, + ) -> Result<(), AtomicCoreError> { + self.storage + .undismiss_health_item_sync(check_name, item_key) + .await + } + // ==================== Chat Operations ==================== /// Create a new conversation diff --git a/crates/atomic-core/src/storage/sqlite/health.rs b/crates/atomic-core/src/storage/sqlite/health.rs index 519332ce..8beddcb9 100644 --- a/crates/atomic-core/src/storage/sqlite/health.rs +++ b/crates/atomic-core/src/storage/sqlite/health.rs @@ -374,7 +374,8 @@ impl SqliteStorage { se.source_atom_id, se.target_atom_id, se.similarity_score, a1.source_url, a1.content, a2.source_url, a2.content, - COUNT(DISTINCT at_a.tag_id) as shared_tag_count + COUNT(DISTINCT at_a.tag_id) as shared_tag_count, + a1.created_at, a2.created_at FROM semantic_edges se JOIN atoms a1 ON se.source_atom_id = a1.id JOIN atoms a2 ON se.target_atom_id = a2.id @@ -396,6 +397,8 @@ impl SqliteStorage { let b_source: Option = row.get(5)?; let b_content: String = row.get(6)?; let shared_tag_count: i32 = row.get(7)?; + let a_created_at: Option = row.get(8)?; + let b_created_at: Option = row.get(9)?; // Skip same-corpus pairs — those are template pollution, not content overlap. let prefix_a = source_prefix(&a_source); @@ -417,6 +420,8 @@ impl SqliteStorage { atom_b_source: b_source, similarity, shared_tag_count, + atom_a_created_at: a_created_at, + atom_b_created_at: b_created_at, }); } } @@ -452,7 +457,8 @@ impl SqliteStorage { se.source_atom_id, se.target_atom_id, se.similarity_score, a1.source_url, a1.content, a2.source_url, a2.content, - COUNT(DISTINCT at_a.tag_id) as shared_tag_count + COUNT(DISTINCT at_a.tag_id) as shared_tag_count, + a1.created_at, a2.created_at FROM semantic_edges se JOIN atoms a1 ON se.source_atom_id = a1.id JOIN atoms a2 ON se.target_atom_id = a2.id @@ -474,12 +480,14 @@ impl SqliteStorage { let b_source: Option = row.get(5)?; let b_content: String = row.get(6)?; let shared_tag_count: i32 = row.get(7)?; + let a_created_at: Option = row.get(8)?; + let b_created_at: Option = row.get(9)?; let a_title = extract_title_preview(&a_content); let b_title = extract_title_preview(&b_content); raw.contradiction_pairs.push(crate::health::ContradictionPairEntry { pair_id: uuid::Uuid::new_v4().to_string(), - atom_a: crate::health::ContradictionAtom { id: a_id, title: a_title, source: a_source }, - atom_b: crate::health::ContradictionAtom { id: b_id, title: b_title, source: b_source }, + atom_a: crate::health::ContradictionAtom { id: a_id, title: a_title, source: a_source, created_at: a_created_at }, + atom_b: crate::health::ContradictionAtom { id: b_id, title: b_title, source: b_source, created_at: b_created_at }, similarity, shared_tag_count, }); diff --git a/package-lock.json b/package-lock.json index 9694866f..078a32e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -49,8 +49,10 @@ "@tauri-apps/plugin-opener": "~2.5.3", "@tauri-apps/plugin-shell": "~2.3.5", "@types/diff": "^7.0.2", + "@types/diff-match-patch": "^1.0.36", "d3-force": "^3.0.0", "diff": "^8.0.4", + "diff-match-patch": "^1.0.5", "graphology": "^0.26.0", "graphology-types": "^0.24.8", "idb-keyval": "^6.2.2", @@ -5032,6 +5034,12 @@ "integrity": "sha512-JSWRMozjFKsGlEjiiKajUjIJVKuKdE3oVy2DNtK+fUo8q82nhFZ2CPQwicAIkXrofahDXrWJ7mjelvZphMS98Q==", "license": "MIT" }, + "node_modules/@types/diff-match-patch": { + "version": "1.0.36", + "resolved": "https://registry.npmjs.org/@types/diff-match-patch/-/diff-match-patch-1.0.36.tgz", + "integrity": "sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -6730,6 +6738,12 @@ "node": ">=0.3.1" } }, + "node_modules/diff-match-patch": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.5.tgz", + "integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==", + "license": "Apache-2.0" + }, "node_modules/dijkstrajs": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/dijkstrajs/-/dijkstrajs-1.0.3.tgz", diff --git a/package.json b/package.json index d49010b2..406dc7d5 100644 --- a/package.json +++ b/package.json @@ -95,8 +95,10 @@ "@tauri-apps/plugin-opener": "~2.5.3", "@tauri-apps/plugin-shell": "~2.3.5", "@types/diff": "^7.0.2", + "@types/diff-match-patch": "^1.0.36", "d3-force": "^3.0.0", "diff": "^8.0.4", + "diff-match-patch": "^1.0.5", "graphology": "^0.26.0", "graphology-types": "^0.24.8", "idb-keyval": "^6.2.2", diff --git a/src/components/dashboard/widgets/HealthReviewModal.tsx b/src/components/dashboard/widgets/HealthReviewModal.tsx index 17e9d8fe..35ed7640 100644 --- a/src/components/dashboard/widgets/HealthReviewModal.tsx +++ b/src/components/dashboard/widgets/HealthReviewModal.tsx @@ -1,8 +1,8 @@ import { useState, useEffect, useCallback, useMemo, useRef } from 'react'; import { createPortal } from 'react-dom'; import { - X, GitMerge, Link, Loader2, CheckCircle, - ChevronDown, ChevronUp, RefreshCw, + X, GitMerge, Loader2, CheckCircle, + ChevronDown, ChevronUp, RefreshCw, ChevronLeft, ChevronRight, Check, Clipboard, } from 'lucide-react'; import { useVirtualizer } from '@tanstack/react-virtual'; import { getTransport } from '../../../lib/transport'; @@ -11,13 +11,15 @@ import { useDatabasesStore } from '../../../stores/databases'; import { NoSourceRow } from './review/NoSourceRow'; import { TagRootlessRow } from './review/TagRootlessRow'; import { BoilerplateAtomRow } from './review/BoilerplateAtomRow'; +import { sourceTrust, relativeAge } from './review/badges'; +import { lineDiff, type DiffPart } from './review/diffUtil'; // ==================== Types ==================== export interface OverlapPair { pair_id: string; - atom_a: { id: string; title: string; source?: string }; - atom_b: { id: string; title: string; source?: string }; + atom_a: { id: string; title: string; source?: string; created_at?: string }; + atom_b: { id: string; title: string; source?: string; created_at?: string }; similarity: number; shared_tag_count: number; available_actions: string[]; @@ -29,7 +31,7 @@ interface AtomDetail { source_url?: string; } -type PairAction = 'merge_with_llm' | 'keep_both'; +type PairAction = 'merge_with_llm' | 'keep_a' | 'keep_b' | 'merge_with_edited_content'; type PairStatus = 'idle' | 'loading' | 'done' | 'error'; @@ -43,8 +45,8 @@ interface BoilerplateEntry { // Contradiction pair interface ContradictionPair { pair_id: string; - atom_a: { id: string; title: string; source?: string }; - atom_b: { id: string; title: string; source?: string }; + atom_a: { id: string; title: string; source?: string; created_at?: string }; + atom_b: { id: string; title: string; source?: string; created_at?: string }; similarity: number; shared_tag_count: number; } @@ -86,12 +88,6 @@ function saveResolved(dbId: string, rec: ResolvedRecord): void { } catch { /* ignore quota errors */ } } -// ==================== Helpers ==================== - -function sourceLabel(source?: string): string { - if (!source) return 'manual'; - try { return new URL(source).hostname; } catch { return source.split('/').slice(0, 2).join('/'); } -} function similarityLabel(s: number): { text: string; color: string } { if (s >= 0.80) return { text: `${(s * 100).toFixed(0)}% overlap`, color: 'text-orange-400' }; @@ -177,12 +173,30 @@ function TabHeader({ // ==================== Overlap pair row ==================== +function DiffView({ a, b }: { a: string; b: string }) { + const parts = useMemo(() => lineDiff(a, b), [a, b]); + return ( + 
    +      {parts.map((p: DiffPart, i: number) => (
+        {p.text}
+      ))}
+
    + ); +} + function PairRow({ pair, - onApply, + onResolve, }: { pair: OverlapPair; - onApply: (pair: OverlapPair, action: PairAction) => Promise; + onResolve: (pair: OverlapPair) => void; }) { const [status, setStatus] = useState('idle'); const [appliedAction, setAppliedAction] = useState(null); @@ -190,46 +204,102 @@ function PairRow({ const [expanded, setExpanded] = useState(false); const [contents, setContents] = useState<[string, string] | null>(null); const [loadingContent, setLoadingContent] = useState(false); + const [mergeOpen, setMergeOpen] = useState(false); + const [mergeDraft, setMergeDraft] = useState(''); + const [diffMode, setDiffMode] = useState(false); const sim = similarityLabel(pair.similarity); - const apply = async (action: PairAction) => { + function buildDraft(a: string, b: string, titleA: string, titleB: string): string { + return `# ${titleA}\n\n${a}\n\n---\n\n# ${titleB}\n\n${b}`.trim(); + } + + const fetchContents = async () => { + setLoadingContent(true); + try { + const [a, b] = await Promise.all([ + getTransport().invoke('get_atom', { id: pair.atom_a.id }), + getTransport().invoke('get_atom', { id: pair.atom_b.id }), + ]); + setContents([a.content, b.content]); + return [a.content, b.content] as [string, string]; + } finally { + setLoadingContent(false); + } + }; + + const toggleExpand = async () => { + if (!expanded && !contents) { + await fetchContents(); + } + setExpanded(v => !v); + }; + + const openMerge = async () => { + let c = contents; + if (!c) { + c = await fetchContents(); + } + if (c) { + setMergeDraft(prev => prev || buildDraft(c![0], c![1], pair.atom_a.title, pair.atom_b.title)); + } + setMergeOpen(true); + }; + + const applyDirect = async (action: 'keep_a' | 'keep_b') => { setStatus('loading'); setAppliedAction(action); setError(null); try { - await onApply(pair, action); + await getTransport().invoke('apply_health_item_fix', { + check: 'content_overlap', + item_id: `${pair.atom_a.id <= pair.atom_b.id ? pair.atom_a.id : pair.atom_b.id}__${pair.atom_a.id <= pair.atom_b.id ? pair.atom_b.id : pair.atom_a.id}`, + action, + }); setStatus('done'); + onResolve(pair); } catch (e) { setStatus('error'); setError(e instanceof Error ? e.message : 'Action failed'); } }; - const toggleExpand = async () => { - if (!expanded && !contents) { - setLoadingContent(true); - try { - const [a, b] = await Promise.all([ - getTransport().invoke('get_atom', { id: pair.atom_a.id }), - getTransport().invoke('get_atom', { id: pair.atom_b.id }), - ]); - setContents([a.content, b.content]); - } finally { - setLoadingContent(false); - } + const applyEditedMerge = async () => { + const aDate = pair.atom_a.created_at ? Date.parse(pair.atom_a.created_at) : 0; + const bDate = pair.atom_b.created_at ? Date.parse(pair.atom_b.created_at) : 0; + const [winner, loser] = aDate >= bDate + ? [pair.atom_a.id, pair.atom_b.id] + : [pair.atom_b.id, pair.atom_a.id]; + setStatus('loading'); + setAppliedAction('merge_with_edited_content'); + setError(null); + try { + await getTransport().invoke('apply_health_item_fix', { + check: 'content_overlap', + item_id: `${pair.atom_a.id <= pair.atom_b.id ? pair.atom_a.id : pair.atom_b.id}__${pair.atom_a.id <= pair.atom_b.id ? pair.atom_b.id : pair.atom_a.id}`, + action: 'merge_with_edited_content', + winner_atom_id: winner, + loser_atom_id: loser, + content: mergeDraft, + }); + setStatus('done'); + onResolve(pair); + } catch (e) { + setStatus('error'); + setError(e instanceof Error ? e.message : 'Merge failed'); } - setExpanded(v => !v); }; if (status === 'done') { - const labels: Record = { + const labels: Record = { merge_with_llm: 'Merged — LLM synthesised both atoms into one', - keep_both: 'Kept both — no changes made', + merge_with_edited_content: 'Merged — edited content applied', + keep_a: 'Kept A; removed B', + keep_b: 'Kept B; removed A', }; return (
    - {labels[appliedAction!]} + {labels[appliedAction!] ?? 'Resolved'}
    ); } @@ -244,6 +314,11 @@ function PairRow({ {pair.shared_tag_count > 0 && ( {pair.shared_tag_count} shared tag{pair.shared_tag_count !== 1 ? 's' : ''} )} + {expanded && contents && ( + + )}
    - {/* Side-by-side content */} + {/* Side-by-side or diff content */} {expanded && contents && ( -
    - {[pair.atom_a, pair.atom_b].map((atom, i) => ( -
    -

    {atom.title}

    - 
    -                  {contents[i as 0 | 1]}
-
    + diffMode + ? + :
    + {[pair.atom_a, pair.atom_b].map((atom, i) => ( +
    +

    {atom.title}

    + 
    +                      {contents[i as 0 | 1]}
+
    +
    + ))}
    - ))} -
    )} {/* Error */} @@ -287,22 +377,51 @@ function PairRow({ {/* Actions */}
    } - label="Merge" - title="LLM synthesises both into one atom, preserving all unique content" - loading={status === 'loading' && appliedAction === 'merge_with_llm'} + icon={} + label="Keep A" + title="Delete the right atom; keep the left one" + loading={status === 'loading' && appliedAction === 'keep_a'} disabled={status === 'loading'} - onClick={() => apply('merge_with_llm')} + onClick={() => applyDirect('keep_a')} /> } - label="Keep both" - title="Leave both atoms — different perspectives on the same topic" - loading={status === 'loading' && appliedAction === 'keep_both'} + icon={} + label="Keep B" + title="Delete the left atom; keep the right one" + loading={status === 'loading' && appliedAction === 'keep_b'} disabled={status === 'loading'} - onClick={() => apply('keep_both')} + onClick={() => applyDirect('keep_b')} + /> + } + label="Merge…" + title="Open an editor to combine both atoms, then delete the loser" + loading={loadingContent && !expanded} + disabled={status === 'loading'} + onClick={openMerge} />
    + + {/* Merge editor */} + {mergeOpen && ( +
    +