diff --git a/README.md b/README.md index 525f083..2e8415e 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,56 @@ -# Gitplant Pass 1 (Desktop-first) +# Gitplant Pass 1.5 (Desktop-first architecture upgrade) -This repository now contains the Pass 1 desktop-first vertical slice defined in `docs/PDF_REVIEW_ARCHITECTURE_PACKAGE.md`. +This repository now includes the architectural upgrade for multi-layer viewing, document transformations, and processing/indexing scaffolding. -## Workspace structure +Design authority: `docs/PDF_REVIEW_ARCHITECTURE_PACKAGE.md`. -- `apps/desktop` – Tauri desktop shell + React/TypeScript UI -- `apps/desktop/src-tauri` – Rust native commands, SQLite schema, managed file storage -- `packages/shared-types` – shared contracts -- `packages/viewer-core` – renderer abstraction interfaces -- `packages/viewer-pdfjs` – PDF.js renderer adapter implementation -- `packages/persistence-core` – persistence gateway interfaces used by UI +## Repo/package structure -## Prerequisites +- `apps/desktop` – Tauri shell + React UI +- `apps/desktop/src-tauri` – Rust commands, SQLite schema, managed storage, transform + processing proof paths +- `packages/shared-types` – domain/shared records for revisions, jobs, extracted text +- `packages/viewer-core` – renderer abstraction + render-scene/layer model +- `packages/viewer-pdfjs` – PDF.js adapter implementation +- `packages/persistence-core` – UI persistence gateway contracts +- `packages/document-transform-core` – transform command/result contracts +- `packages/document-transform-pdflib` – adapter slot (desktop implementation currently in Rust service) +- `packages/processing-core` – processing job and OCR provider contracts +- `packages/text-extraction-core` – text extraction provider contracts -- Node.js 20+ -- Rust stable toolchain -- Tauri system dependencies: https://v2.tauri.app/start/prerequisites/ +## What changed + +- Viewer architecture now models stacked render layers (`base_pdf`, `overlay_pdf`, future markup/selection overlays). +- Transformations are isolated behind dedicated command boundary and create **derived revisions**. +- Processing pipeline tracks jobs and stores extracted page text by revision/page. +- Schema supports immutable originals, revision lineage, processing jobs, extracted text, and audit events. +- Desktop app includes minimal dev scaffolding buttons to trigger text extraction and extract-page transformation proof path. + +## Implemented now (real paths) + +1. Import + open PDFs (existing behavior still supported). +2. Extract page range to a new derived revision (managed storage + metadata persistence). +3. Trigger text extraction job for current revision and persist per-page text. + +## Scaffolded for later + +- OCR provider implementation (boundary and job type already in place) +- Full transform UI for delete/insert/reorder/combine +- Native renderer adapter implementation +- Full markup/selection overlay rendering ## Commands From repository root: -- Dev (one command): `npm run desktop:dev` +- Dev (desktop): `npm run desktop:dev` - Desktop build: `npm run desktop:build` - Tests: `npm test` - Typecheck: `npm run typecheck` -## Pass 1 manual verification checklist +## Manual verification checklist 1. Run `npm run desktop:dev` and confirm desktop window opens. -2. Click **Import PDF**, pick a local `.pdf` through native picker. -3. Confirm document appears in viewer and page renders. -4. Use **Prev/Next** and **Zoom In/Zoom Out/Fit Width**. -5. Confirm imported document appears in **Recent Documents**. -6. Restart app and confirm recent item reopens. - -## Deferred to Pass 2+ - -- Markup tools/redlines/comments/workflow -- Collaboration/sync/export pipeline -- Native renderer adapter -- Advanced performance optimization and tile rendering +2. Import a PDF and confirm page rendering still works. +3. Click **Extract page 1 to derived revision** and confirm no crash + viewer shows comparison-capable scene layer info. +4. Click **Trigger text extraction** and confirm extracted row count updates. +5. Restart app and confirm recent document still opens. diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 92591cd..72f1ca1 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -17,7 +17,11 @@ "@gitplant/viewer-pdfjs": "0.1.0", "@tauri-apps/api": "^2.0.0", "react": "^18.3.1", - "react-dom": "^18.3.1" + "react-dom": "^18.3.1", + "@gitplant/viewer-core": "0.1.0", + "@gitplant/processing-core": "0.1.0", + "@gitplant/document-transform-core": "0.1.0", + "@gitplant/text-extraction-core": "0.1.0" }, "devDependencies": { "@tauri-apps/cli": "^2.0.0", @@ -31,4 +35,4 @@ "vite": "^6.0.5", "vitest": "^2.1.8" } -} \ No newline at end of file +} diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 4cfeadf..85e23f9 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -15,6 +15,11 @@ serde = { version = "1", features = ["derive"] } tauri = { version = "2", features = [] } uuid = { version = "1", features = ["v4", "serde"] } rfd = "0.15" +lopdf = "0.35" +serde_json = "1" [build-dependencies] tauri-build = { version = "2", features = [] } + +[dev-dependencies] +tempfile = "3" diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 9838198..5245f69 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -1,5 +1,12 @@ -use std::{fs, path::{Path, PathBuf}}; +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, +}; + +use anyhow::anyhow; use chrono::Utc; +use lopdf::Document as LoDocument; use rusqlite::{params, Connection}; use serde::Serialize; use tauri::Manager; @@ -9,11 +16,22 @@ use uuid::Uuid; #[serde(rename_all = "camelCase")] struct ImportedDocumentPayload { document_id: String, + revision_id: String, title: String, managed_file_path: String, file_size_bytes: u64, } +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct DerivedRevisionPayload { + document_id: String, + source_revision_id: String, + derived_revision_id: String, + managed_file_path: String, + page_count: i64, +} + #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct RecentDocumentView { @@ -22,6 +40,31 @@ struct RecentDocumentView { managed_file_path: String, opened_at: String, page_count: Option, + active_revision_id: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ProcessingJobRecord { + id: String, + revision_id: String, + job_type: String, + status: String, + payload_json: Option, + error_message: Option, + created_at: String, + started_at: Option, + completed_at: Option, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ExtractedPageTextRecord { + id: String, + revision_id: String, + page_number: i64, + text_content: String, + extracted_at: String, } fn db_path(app: &tauri::AppHandle) -> anyhow::Result { @@ -46,11 +89,13 @@ fn init_schema(conn: &Connection) -> anyhow::Result<()> { revision_number INTEGER NOT NULL, managed_file_path TEXT NOT NULL, original_file_name TEXT NOT NULL, - source_path TEXT, page_count INTEGER, file_size_bytes INTEGER NOT NULL, imported_at TEXT NOT NULL, - FOREIGN KEY(document_id) REFERENCES documents(id) + source_revision_id TEXT, + derivation_type TEXT, + FOREIGN KEY(document_id) REFERENCES documents(id), + FOREIGN KEY(source_revision_id) REFERENCES document_revisions(id) ); CREATE TABLE IF NOT EXISTS recent_documents ( id TEXT PRIMARY KEY, @@ -58,7 +103,35 @@ fn init_schema(conn: &Connection) -> anyhow::Result<()> { opened_at TEXT NOT NULL, FOREIGN KEY(document_id) REFERENCES documents(id) ); - " + CREATE TABLE IF NOT EXISTS processing_jobs ( + id TEXT PRIMARY KEY, + revision_id TEXT NOT NULL, + job_type TEXT NOT NULL, + status TEXT NOT NULL, + payload_json TEXT, + error_message TEXT, + created_at TEXT NOT NULL, + started_at TEXT, + completed_at TEXT, + FOREIGN KEY(revision_id) REFERENCES document_revisions(id) + ); + CREATE TABLE IF NOT EXISTS extracted_page_text ( + id TEXT PRIMARY KEY, + revision_id TEXT NOT NULL, + page_number INTEGER NOT NULL, + text_content TEXT NOT NULL, + extracted_at TEXT NOT NULL, + FOREIGN KEY(revision_id) REFERENCES document_revisions(id) + ); + CREATE TABLE IF NOT EXISTS audit_events ( + id TEXT PRIMARY KEY, + entity_type TEXT NOT NULL, + entity_id TEXT NOT NULL, + event_type TEXT NOT NULL, + payload_json TEXT, + occurred_at TEXT NOT NULL + ); + ", )?; Ok(()) } @@ -69,71 +142,365 @@ fn open_conn(app: &tauri::AppHandle) -> anyhow::Result, +) -> anyhow::Result<()> { + conn.execute( + "INSERT INTO audit_events (id,entity_type,entity_id,event_type,payload_json,occurred_at) VALUES (?1,?2,?3,?4,?5,?6)", + params![ + Uuid::new_v4().to_string(), + entity_type, + entity_id, + event_type, + payload_json, + Utc::now().to_rfc3339() + ], + )?; + Ok(()) +} + +fn latest_revision_for_document(conn: &Connection, document_id: &str) -> Result<(String, String, String), String> { + conn.query_row( + "SELECT d.title,dr.managed_file_path,dr.id FROM documents d JOIN document_revisions dr ON dr.document_id=d.id WHERE d.id=?1 ORDER BY dr.revision_number DESC LIMIT 1", + params![document_id], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .map_err(|e| e.to_string()) +} + +fn revision_row(conn: &Connection, revision_id: &str) -> Result<(String, String, String), String> { + conn.query_row( + "SELECT document_id, managed_file_path, original_file_name FROM document_revisions WHERE id=?1", + params![revision_id], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .map_err(|e| e.to_string()) +} + +fn extract_pages(source_path: &str, start_page: u32, end_page: u32, output_path: &Path) -> anyhow::Result { + let mut source = LoDocument::load(source_path)?; + let pages: BTreeMap = source.get_pages(); + let keep: Vec = pages + .keys() + .copied() + .filter(|p| *p >= start_page && *p <= end_page) + .collect(); + if keep.is_empty() { + return Err(anyhow!("No pages in selected range")); + } + source.extract_pages(&keep); + source.prune_objects(); + source.save(output_path)?; + Ok(keep.len()) +} + #[tauri::command] fn import_pdf_from_picker(app: tauri::AppHandle) -> Result { - let selected = rfd::FileDialog::new().add_filter("PDF", &["pdf"]).pick_file().ok_or("No file selected")?; - if selected.extension().and_then(|e| e.to_str()).map(|s| s.to_lowercase()) != Some("pdf".into()) { + let selected = rfd::FileDialog::new() + .add_filter("PDF", &["pdf"]) + .pick_file() + .ok_or("No file selected")?; + if selected + .extension() + .and_then(|e| e.to_str()) + .map(|s| s.to_lowercase()) + != Some("pdf".into()) + { return Err("Invalid file type. Please select a PDF.".into()); } + let doc_id = Uuid::new_v4().to_string(); let rev_id = Uuid::new_v4().to_string(); let now = Utc::now().to_rfc3339(); - let name = selected.file_stem().and_then(|s| s.to_str()).unwrap_or("Untitled").to_string(); - let file_name = selected.file_name().and_then(|s| s.to_str()).unwrap_or("document.pdf").to_string(); - let managed_path = storage_dir(&app).map_err(|e| e.to_string())?.join(format!("{}-{}", doc_id, file_name)); + let name = selected + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("Untitled") + .to_string(); + let file_name = selected + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("document.pdf") + .to_string(); + let managed_path = storage_dir(&app) + .map_err(|e| e.to_string())? + .join(format!("{}-{}", doc_id, file_name)); fs::copy(&selected, &managed_path).map_err(|e| format!("Import failure: {e}"))?; let file_size = fs::metadata(&managed_path).map_err(|e| e.to_string())?.len(); let conn = open_conn(&app).map_err(|e| format!("DB initialization failure: {e}"))?; - conn.execute("INSERT INTO documents (id, title, created_at, updated_at) VALUES (?1, ?2, ?3, ?3)", params![doc_id, name, now]).map_err(|e| e.to_string())?; - conn.execute("INSERT INTO document_revisions (id, document_id, revision_number, managed_file_path, original_file_name, source_path, page_count, file_size_bytes, imported_at) VALUES (?1, ?2, 1, ?3, ?4, ?5, NULL, ?6, ?7)", - params![rev_id, doc_id, managed_path.to_string_lossy().to_string(), file_name, selected.to_string_lossy().to_string(), file_size as i64, now]).map_err(|e| e.to_string())?; - conn.execute("INSERT INTO recent_documents (id, document_id, opened_at) VALUES (?1, ?2, ?3)", params![Uuid::new_v4().to_string(), doc_id, now]).map_err(|e| e.to_string())?; + conn.execute( + "INSERT INTO documents (id, title, created_at, updated_at) VALUES (?1, ?2, ?3, ?3)", + params![doc_id, name, now], + ) + .map_err(|e| e.to_string())?; + conn.execute( + "INSERT INTO document_revisions (id, document_id, revision_number, managed_file_path, original_file_name, page_count, file_size_bytes, imported_at, source_revision_id, derivation_type) VALUES (?1, ?2, 1, ?3, ?4, NULL, ?5, ?6, NULL, 'imported_original')", + params![rev_id, doc_id, managed_path.to_string_lossy().to_string(), file_name, file_size as i64, now], + ).map_err(|e| e.to_string())?; + conn.execute( + "INSERT INTO recent_documents (id, document_id, opened_at) VALUES (?1, ?2, ?3)", + params![Uuid::new_v4().to_string(), doc_id, now], + ) + .map_err(|e| e.to_string())?; + insert_audit_event( + &conn, + "document_revision", + &rev_id, + "document_revision.imported", + None, + ) + .map_err(|e| e.to_string())?; - Ok(ImportedDocumentPayload { document_id: doc_id, title: name, managed_file_path: managed_path.to_string_lossy().to_string(), file_size_bytes: file_size }) + Ok(ImportedDocumentPayload { + document_id: doc_id, + revision_id: rev_id, + title: name, + managed_file_path: managed_path.to_string_lossy().to_string(), + file_size_bytes: file_size, + }) } #[tauri::command] fn list_recent_documents(app: tauri::AppHandle) -> Result, String> { let conn = open_conn(&app).map_err(|e| e.to_string())?; - let mut stmt = conn.prepare("SELECT d.id,d.title,dr.managed_file_path,r.opened_at,dr.page_count FROM recent_documents r JOIN documents d ON d.id=r.document_id JOIN document_revisions dr ON dr.document_id=d.id ORDER BY r.opened_at DESC LIMIT 20").map_err(|e| e.to_string())?; - let rows = stmt.query_map([], |row| Ok(RecentDocumentView { - document_id: row.get(0)?, title: row.get(1)?, managed_file_path: row.get(2)?, opened_at: row.get(3)?, page_count: row.get(4)? - })).map_err(|e| e.to_string())?; + let mut stmt = conn + .prepare("SELECT d.id,d.title,dr.managed_file_path,r.opened_at,dr.page_count,dr.id FROM recent_documents r JOIN documents d ON d.id=r.document_id JOIN document_revisions dr ON dr.document_id=d.id AND dr.revision_number=(SELECT MAX(revision_number) FROM document_revisions drr WHERE drr.document_id=d.id) ORDER BY r.opened_at DESC LIMIT 20") + .map_err(|e| e.to_string())?; + let rows = stmt + .query_map([], |row| { + Ok(RecentDocumentView { + document_id: row.get(0)?, + title: row.get(1)?, + managed_file_path: row.get(2)?, + opened_at: row.get(3)?, + page_count: row.get(4)?, + active_revision_id: row.get(5)?, + }) + }) + .map_err(|e| e.to_string())?; Ok(rows.filter_map(Result::ok).collect()) } -fn latest_path(conn: &Connection, document_id: &str) -> Result<(String, String), String> { - conn.query_row("SELECT d.title,dr.managed_file_path FROM documents d JOIN document_revisions dr ON dr.document_id=d.id WHERE d.id=?1 ORDER BY dr.revision_number DESC LIMIT 1", params![document_id], |r| Ok((r.get(0)?, r.get(1)?))).map_err(|e| e.to_string()) -} - #[tauri::command] fn open_document(app: tauri::AppHandle, document_id: String) -> Result { let conn = open_conn(&app).map_err(|e| e.to_string())?; - let (title, managed_file_path) = latest_path(&conn, &document_id)?; - conn.execute("INSERT INTO recent_documents (id, document_id, opened_at) VALUES (?1, ?2, ?3)", params![Uuid::new_v4().to_string(), document_id, Utc::now().to_rfc3339()]).map_err(|e| e.to_string())?; - let file_size_bytes = fs::metadata(&managed_file_path).map_err(|e| e.to_string())?.len(); - Ok(ImportedDocumentPayload { document_id, title, managed_file_path, file_size_bytes }) + let (title, managed_file_path, revision_id) = latest_revision_for_document(&conn, &document_id)?; + conn.execute( + "INSERT INTO recent_documents (id, document_id, opened_at) VALUES (?1, ?2, ?3)", + params![Uuid::new_v4().to_string(), document_id, Utc::now().to_rfc3339()], + ) + .map_err(|e| e.to_string())?; + let file_size_bytes = fs::metadata(&managed_file_path) + .map_err(|e| e.to_string())? + .len(); + Ok(ImportedDocumentPayload { + document_id, + revision_id, + title, + managed_file_path, + file_size_bytes, + }) } #[tauri::command] -fn read_document_bytes(app: tauri::AppHandle, document_id: String) -> Result, String> { +fn read_document_bytes( + app: tauri::AppHandle, + document_id: String, + revision_id: Option, +) -> Result, String> { let conn = open_conn(&app).map_err(|e| e.to_string())?; - let (_, managed_file_path) = latest_path(&conn, &document_id)?; + let managed_file_path = if let Some(target_revision) = revision_id { + let (_, path, _) = revision_row(&conn, &target_revision)?; + path + } else { + let (_, path, _) = latest_revision_for_document(&conn, &document_id)?; + path + }; fs::read(Path::new(&managed_file_path)).map_err(|e| e.to_string()) } #[tauri::command] -fn update_page_count(app: tauri::AppHandle, document_id: String, page_count: i64) -> Result<(), String> { +fn update_page_count(app: tauri::AppHandle, revision_id: String, page_count: i64) -> Result<(), String> { let conn = open_conn(&app).map_err(|e| e.to_string())?; - conn.execute("UPDATE document_revisions SET page_count=?1 WHERE document_id=?2 AND revision_number=1", params![page_count, document_id]).map_err(|e| e.to_string())?; + conn.execute( + "UPDATE document_revisions SET page_count=?1 WHERE id=?2", + params![page_count, revision_id], + ) + .map_err(|e| e.to_string())?; Ok(()) } +#[tauri::command] +fn extract_pages_to_derived_revision( + app: tauri::AppHandle, + revision_id: String, + start_page: u32, + end_page: u32, +) -> Result { + let conn = open_conn(&app).map_err(|e| e.to_string())?; + let (document_id, source_path, original_name) = revision_row(&conn, &revision_id)?; + let next_revision_number: i64 = conn + .query_row( + "SELECT COALESCE(MAX(revision_number),0) + 1 FROM document_revisions WHERE document_id=?1", + params![document_id], + |r| r.get(0), + ) + .map_err(|e| e.to_string())?; + + let derived_revision_id = Uuid::new_v4().to_string(); + let out_name = format!("{}-derived-{}", derived_revision_id, original_name); + let output_path = storage_dir(&app) + .map_err(|e| e.to_string())? + .join(out_name); + + insert_audit_event( + &conn, + "document_transformation", + &derived_revision_id, + "transformation.extract_pages.requested", + Some(format!( + "{{\"sourceRevisionId\":\"{}\",\"startPage\":{},\"endPage\":{}}}", + revision_id, start_page, end_page + )), + ) + .map_err(|e| e.to_string())?; + + let page_count = extract_pages(&source_path, start_page, end_page, &output_path).map_err(|e| e.to_string())? as i64; + let file_size = fs::metadata(&output_path).map_err(|e| e.to_string())?.len() as i64; + + conn.execute( + "INSERT INTO document_revisions (id, document_id, revision_number, managed_file_path, original_file_name, page_count, file_size_bytes, imported_at, source_revision_id, derivation_type) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 'extract_pages')", + params![derived_revision_id, document_id, next_revision_number, output_path.to_string_lossy().to_string(), original_name, page_count, file_size, Utc::now().to_rfc3339(), revision_id], + ) + .map_err(|e| e.to_string())?; + + insert_audit_event( + &conn, + "document_transformation", + &derived_revision_id, + "transformation.extract_pages.completed", + Some(format!("{{\"pageCount\":{}}}", page_count)), + ) + .map_err(|e| e.to_string())?; + + Ok(DerivedRevisionPayload { + document_id, + source_revision_id: revision_id, + derived_revision_id, + managed_file_path: output_path.to_string_lossy().to_string(), + page_count, + }) +} + +#[tauri::command] +fn trigger_text_extraction(app: tauri::AppHandle, revision_id: String) -> Result { + let conn = open_conn(&app).map_err(|e| e.to_string())?; + let (_, source_path, _) = revision_row(&conn, &revision_id)?; + let now = Utc::now().to_rfc3339(); + let job_id = Uuid::new_v4().to_string(); + conn.execute( + "INSERT INTO processing_jobs (id,revision_id,job_type,status,payload_json,error_message,created_at,started_at,completed_at) VALUES (?1,?2,'text_extraction','running',NULL,NULL,?3,?3,NULL)", + params![job_id, revision_id, now], + ) + .map_err(|e| e.to_string())?; + conn.execute("DELETE FROM extracted_page_text WHERE revision_id=?1", params![revision_id]) + .map_err(|e| e.to_string())?; + + let extraction = (|| -> anyhow::Result<()> { + let doc = LoDocument::load(&source_path)?; + let pages = doc.get_pages(); + for page_num in pages.keys() { + let text = doc.extract_text(&[*page_num]).unwrap_or_default(); + conn.execute( + "INSERT INTO extracted_page_text (id,revision_id,page_number,text_content,extracted_at) VALUES (?1,?2,?3,?4,?5)", + params![Uuid::new_v4().to_string(), revision_id, *page_num as i64, text, Utc::now().to_rfc3339()], + )?; + } + Ok(()) + })(); + + match extraction { + Ok(()) => { + let done = Utc::now().to_rfc3339(); + conn.execute( + "UPDATE processing_jobs SET status='completed', completed_at=?1 WHERE id=?2", + params![done, job_id], + ) + .map_err(|e| e.to_string())?; + insert_audit_event( + &conn, + "processing_job", + &job_id, + "processing.text_extraction.completed", + None, + ) + .map_err(|e| e.to_string())?; + Ok(ProcessingJobRecord { + id: job_id, + revision_id, + job_type: "text_extraction".into(), + status: "completed".into(), + payload_json: None, + error_message: None, + created_at: now.clone(), + started_at: Some(now), + completed_at: Some(done), + }) + } + Err(err) => { + let done = Utc::now().to_rfc3339(); + let error_message = err.to_string(); + conn.execute( + "UPDATE processing_jobs SET status='failed', completed_at=?1, error_message=?2 WHERE id=?3", + params![done, error_message, job_id], + ) + .map_err(|e| e.to_string())?; + Err(error_message) + } + } +} + +#[tauri::command] +fn list_extracted_page_text( + app: tauri::AppHandle, + revision_id: String, +) -> Result, String> { + let conn = open_conn(&app).map_err(|e| e.to_string())?; + let mut stmt = conn + .prepare("SELECT id,revision_id,page_number,text_content,extracted_at FROM extracted_page_text WHERE revision_id=?1 ORDER BY page_number") + .map_err(|e| e.to_string())?; + let rows = stmt + .query_map(params![revision_id], |r| { + Ok(ExtractedPageTextRecord { + id: r.get(0)?, + revision_id: r.get(1)?, + page_number: r.get(2)?, + text_content: r.get(3)?, + extracted_at: r.get(4)?, + }) + }) + .map_err(|e| e.to_string())?; + Ok(rows.filter_map(Result::ok).collect()) +} + #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { tauri::Builder::default() - .invoke_handler(tauri::generate_handler![import_pdf_from_picker, list_recent_documents, open_document, read_document_bytes, update_page_count]) + .invoke_handler(tauri::generate_handler![ + import_pdf_from_picker, + list_recent_documents, + open_document, + read_document_bytes, + update_page_count, + extract_pages_to_derived_revision, + trigger_text_extraction, + list_extracted_page_text + ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); } @@ -143,14 +510,74 @@ mod tests { use super::*; #[test] - fn sqlite_schema_supports_document_and_recent_persistence() { + fn sqlite_schema_supports_new_architecture_entities() { let conn = Connection::open_in_memory().unwrap(); init_schema(&conn).unwrap(); - conn.execute("INSERT INTO documents (id,title,created_at,updated_at) VALUES ('d1','Doc','t','t')", []).unwrap(); - conn.execute("INSERT INTO document_revisions (id,document_id,revision_number,managed_file_path,original_file_name,source_path,page_count,file_size_bytes,imported_at) VALUES ('r1','d1',1,'/tmp/a.pdf','a.pdf','/src/a.pdf',3,99,'t')", []).unwrap(); - conn.execute("INSERT INTO recent_documents (id,document_id,opened_at) VALUES ('x','d1','t')", []).unwrap(); - let mut stmt = conn.prepare("SELECT COUNT(*) FROM recent_documents").unwrap(); - let count: i64 = stmt.query_row([], |r| r.get(0)).unwrap(); + conn.execute( + "INSERT INTO documents (id,title,created_at,updated_at) VALUES ('d1','Doc','t','t')", + [], + ) + .unwrap(); + conn.execute("INSERT INTO document_revisions (id,document_id,revision_number,managed_file_path,original_file_name,page_count,file_size_bytes,imported_at,source_revision_id,derivation_type) VALUES ('r1','d1',1,'/tmp/a.pdf','a.pdf',3,99,'t',NULL,'imported_original')", []) + .unwrap(); + conn.execute("INSERT INTO document_revisions (id,document_id,revision_number,managed_file_path,original_file_name,page_count,file_size_bytes,imported_at,source_revision_id,derivation_type) VALUES ('r2','d1',2,'/tmp/b.pdf','a.pdf',1,25,'t','r1','extract_pages')", []) + .unwrap(); + conn.execute("INSERT INTO processing_jobs (id,revision_id,job_type,status,payload_json,error_message,created_at,started_at,completed_at) VALUES ('j1','r2','text_extraction','completed',NULL,NULL,'t','t','t')", []) + .unwrap(); + conn.execute("INSERT INTO extracted_page_text (id,revision_id,page_number,text_content,extracted_at) VALUES ('e1','r2',1,'hello','t')", []) + .unwrap(); + let rel: String = conn + .query_row( + "SELECT source_revision_id FROM document_revisions WHERE id='r2'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(rel, "r1"); + } + + #[test] + fn extract_pages_creates_output_pdf() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("source.pdf"); + let output = dir.path().join("out.pdf"); + fs::write( + &source, + b"%PDF-1.4 +1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj +2 0 obj << /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >> endobj +3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] >> endobj +4 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] >> endobj +xref +0 5 +0000000000 65535 f +0000000010 00000 n +0000000060 00000 n +0000000130 00000 n +0000000190 00000 n +trailer << /Root 1 0 R /Size 5 >> +startxref +250 +%%EOF", + ) + .unwrap(); + + let count = extract_pages(source.to_str().unwrap(), 1, 1, &output).unwrap(); assert_eq!(count, 1); + assert!(output.exists()); + } + + #[test] + fn processing_and_text_tables_support_status_updates() { + let conn = Connection::open_in_memory().unwrap(); + init_schema(&conn).unwrap(); + conn.execute("INSERT INTO documents (id,title,created_at,updated_at) VALUES ('d1','Doc','t','t')",[]).unwrap(); + conn.execute("INSERT INTO document_revisions (id,document_id,revision_number,managed_file_path,original_file_name,page_count,file_size_bytes,imported_at,source_revision_id,derivation_type) VALUES ('r1','d1',1,'/tmp/a.pdf','a.pdf',1,1,'t',NULL,'imported_original')",[]).unwrap(); + conn.execute("INSERT INTO processing_jobs (id,revision_id,job_type,status,payload_json,error_message,created_at,started_at,completed_at) VALUES ('j1','r1','ocr','pending',NULL,NULL,'t',NULL,NULL)",[]).unwrap(); + conn.execute("UPDATE processing_jobs SET status='running', started_at='t2' WHERE id='j1'",[]).unwrap(); + conn.execute("INSERT INTO extracted_page_text (id,revision_id,page_number,text_content,extracted_at) VALUES ('x1','r1',1,'abc','t3')",[]).unwrap(); + let status: String = conn.query_row("SELECT status FROM processing_jobs WHERE id='j1'", [], |r| r.get(0)).unwrap(); + assert_eq!(status, "running"); } + } diff --git a/apps/desktop/src/App.test.tsx b/apps/desktop/src/App.test.tsx index a3f7f83..71214de 100644 --- a/apps/desktop/src/App.test.tsx +++ b/apps/desktop/src/App.test.tsx @@ -8,7 +8,10 @@ vi.mock('./lib/tauriGateway', () => ({ importPdfFromPicker: vi.fn(), openDocument: vi.fn(), readDocumentBytes: vi.fn(), - updatePageCount: vi.fn() + updatePageCount: vi.fn(), + extractPagesToDerivedRevision: vi.fn(), + triggerTextExtraction: vi.fn(), + listExtractedPageText: vi.fn().mockResolvedValue([]) } })); @@ -17,5 +20,6 @@ describe('App smoke', () => { render(); expect(screen.getByText('Gitplant Desktop')).toBeInTheDocument(); expect(await screen.findByText('No recent documents.')).toBeInTheDocument(); + expect(screen.getByText('Dev scaffolding')).toBeInTheDocument(); }); }); diff --git a/apps/desktop/src/App.tsx b/apps/desktop/src/App.tsx index c0b9a52..f5a8dd3 100644 --- a/apps/desktop/src/App.tsx +++ b/apps/desktop/src/App.tsx @@ -2,24 +2,32 @@ import { useEffect, useState } from 'react'; import { tauriGateway } from './lib/tauriGateway'; import { PdfViewer } from './components/PdfViewer'; import { ErrorBoundary } from './components/ErrorBoundary'; -import type { RecentDocumentView } from '@gitplant/shared-types'; +import type { ExtractedPageTextRecord, RecentDocumentView } from '@gitplant/shared-types'; export function App() { const [recent, setRecent] = useState([]); const [currentId, setCurrentId] = useState(null); + const [currentRevisionId, setCurrentRevisionId] = useState(null); const [bytes, setBytes] = useState(null); + const [overlayBytes, setOverlayBytes] = useState(null); const [title, setTitle] = useState(''); const [error, setError] = useState(''); + const [extractedText, setExtractedText] = useState([]); const refreshRecent = async () => setRecent(await tauriGateway.listRecentDocuments()); - useEffect(() => { void refreshRecent(); }, []); + useEffect(() => { + void refreshRecent(); + }, []); const openDocument = async (documentId: string) => { const data = await tauriGateway.openDocument(documentId); - const raw = await tauriGateway.readDocumentBytes(documentId); + const raw = await tauriGateway.readDocumentBytes(documentId, data.revisionId); setCurrentId(documentId); + setCurrentRevisionId(data.revisionId); setTitle(data.title); setBytes(new Uint8Array(raw)); + setOverlayBytes(null); + setExtractedText([]); await refreshRecent(); }; @@ -33,15 +41,61 @@ export function App() { } }; - return
-

Gitplant Desktop

- {error &&
{error}
} - -
{ if (currentId) void tauriGateway.updatePageCount(currentId, n); }} />
-
; + const runTextExtraction = async () => { + if (!currentRevisionId) return; + await tauriGateway.triggerTextExtraction(currentRevisionId); + const rows = await tauriGateway.listExtractedPageText(currentRevisionId); + setExtractedText(rows); + }; + + const runExtractProof = async () => { + if (!currentRevisionId || !currentId) return; + const derived = await tauriGateway.extractPagesToDerivedRevision(currentRevisionId, 1, 1); + const base = await tauriGateway.readDocumentBytes(currentId, derived.sourceRevisionId); + const overlay = await tauriGateway.readDocumentBytes(currentId, derived.derivedRevisionId); + setBytes(new Uint8Array(base)); + setOverlayBytes(new Uint8Array(overlay)); + setCurrentRevisionId(derived.derivedRevisionId); + await refreshRecent(); + }; + + return ( + +
+
+

Gitplant Desktop

+ +
+ {error &&
{error}
} + +
+

Dev scaffolding

+ + +
Extracted page text rows: {extractedText.length}
+
+
+ { + if (currentRevisionId) void tauriGateway.updatePageCount(currentRevisionId, n); + }} + /> +
+
+ + ); } diff --git a/apps/desktop/src/components/PdfViewer.test.tsx b/apps/desktop/src/components/PdfViewer.test.tsx index 8ab73e9..399128c 100644 --- a/apps/desktop/src/components/PdfViewer.test.tsx +++ b/apps/desktop/src/components/PdfViewer.test.tsx @@ -1,9 +1,27 @@ import { render, screen } from '@testing-library/react'; import { PdfViewer } from './PdfViewer'; +import { createBaseRenderScene, withOverlayPdfLayer } from '@gitplant/viewer-core'; +import fs from 'node:fs'; +import path from 'node:path'; describe('PdfViewer states', () => { it('shows empty state', () => { render(); expect(screen.getByText('Select a PDF to view.')).toBeInTheDocument(); }); + + it('supports base scene then optional overlay scene', () => { + const base = createBaseRenderScene(1, 1); + expect(base.layers).toHaveLength(1); + expect(base.layers[0].kind).toBe('base_pdf'); + const compare = withOverlayPdfLayer(base, 'r2'); + expect(compare.layers).toHaveLength(2); + expect(compare.layers[1].kind).toBe('overlay_pdf'); + }); + + it('ui-level code avoids direct pdfjs imports', () => { + const appSource = fs.readFileSync(path.resolve(__dirname, '../App.tsx'), 'utf8'); + expect(appSource).not.toContain('pdfjs-dist'); + expect(appSource).not.toContain('@gitplant/viewer-pdfjs'); + }); }); diff --git a/apps/desktop/src/components/PdfViewer.tsx b/apps/desktop/src/components/PdfViewer.tsx index 01dff12..b378949 100644 --- a/apps/desktop/src/components/PdfViewer.tsx +++ b/apps/desktop/src/components/PdfViewer.tsx @@ -1,54 +1,85 @@ import { useEffect, useMemo, useState } from 'react'; +import { createBaseRenderScene, withOverlayPdfLayer } from '@gitplant/viewer-core'; import { PdfjsRendererAdapter } from '@gitplant/viewer-pdfjs'; -type Props = { bytes: Uint8Array | null; title: string; onPageCount?: (n: number) => void }; +type Props = { + bytes: Uint8Array | null; + title: string; + overlayBytes?: Uint8Array | null; + onPageCount?: (n: number) => void; +}; -export function PdfViewer({ bytes, title, onPageCount }: Props) { +export function PdfViewer({ bytes, title, overlayBytes, onPageCount }: Props) { const renderer = useMemo(() => new PdfjsRendererAdapter(), []); const [page, setPage] = useState(1); const [pageCount, setPageCount] = useState(0); const [zoom, setZoom] = useState(1); const [image, setImage] = useState(''); - const [state, setState] = useState<'idle'|'loading'|'error'>('idle'); + const [state, setState] = useState<'idle' | 'loading' | 'error'>('idle'); + const [scene, setScene] = useState(() => createBaseRenderScene(1, 1)); - useEffect(() => { void (async () => { - if (!bytes) return; - setState('loading'); - try { - await renderer.openDocument(bytes); - const count = await renderer.getPageCount(); - setPage(1); setPageCount(count); onPageCount?.(count); - const render = await renderer.renderPage(1, { scale: zoom }); - setImage(render.imageDataUrl); - setState('idle'); - } catch { - setState('error'); - } - })(); return () => { void renderer.closeDocument(); }; }, [bytes]); + useEffect(() => { + void (async () => { + if (!bytes) return; + setState('loading'); + try { + await renderer.openDocument(bytes); + const count = await renderer.getPageCount(); + const initialScene = overlayBytes ? withOverlayPdfLayer(createBaseRenderScene(1, zoom), 'overlay') : createBaseRenderScene(1, zoom); + setScene(initialScene); + setPage(1); + setPageCount(count); + onPageCount?.(count); + const render = await renderer.renderLayer(initialScene.layers[0], 1, { scale: zoom }); + setImage(render.imageDataUrl); + setState('idle'); + } catch { + setState('error'); + } + })(); + return () => { + void renderer.closeDocument(); + }; + }, [bytes, overlayBytes]); - useEffect(() => { void (async () => { - if (!bytes || state === 'error' || pageCount === 0) return; - setState('loading'); - try { - const render = await renderer.renderPage(page, { scale: zoom }); - setImage(render.imageDataUrl); - setState('idle'); - } catch { setState('error'); } - })(); }, [page, zoom]); + useEffect(() => { + void (async () => { + if (!bytes || state === 'error' || pageCount === 0) return; + setState('loading'); + try { + const nextScene = { + ...scene, + pageNumber: page, + viewport: { ...scene.viewport, scale: zoom } + }; + setScene(nextScene); + const render = await renderer.renderLayer(nextScene.layers[0], page, { scale: zoom }); + setImage(render.imageDataUrl); + setState('idle'); + } catch { + setState('error'); + } + })(); + }, [page, zoom]); if (!bytes) return
Select a PDF to view.
; if (state === 'error') return
Failed to render document.
; - return
-

{title}

+ return (
- - {page}/{pageCount} - - - - +

{title}

+
+ + + {page}/{pageCount} + + + + + +
+
Scene layers: {scene.layers.map((l) => l.kind).join(', ')}
+ {state === 'loading' ?
Loading...
: pdf page} +
- {state === 'loading' ?
Loading...
: pdf page} -
-
; + ); } diff --git a/apps/desktop/src/lib/tauriGateway.ts b/apps/desktop/src/lib/tauriGateway.ts index 13c6e7d..61e63b8 100644 --- a/apps/desktop/src/lib/tauriGateway.ts +++ b/apps/desktop/src/lib/tauriGateway.ts @@ -1,11 +1,15 @@ import { invoke } from '@tauri-apps/api/core'; -import type { PersistenceGateway, ImportedDocumentPayload } from '@gitplant/persistence-core'; -import type { RecentDocumentView } from '@gitplant/shared-types'; +import type { PersistenceGateway, ImportedDocumentPayload, DerivedRevisionPayload } from '@gitplant/persistence-core'; +import type { ExtractedPageTextRecord, ProcessingJobRecord, RecentDocumentView } from '@gitplant/shared-types'; export const tauriGateway: PersistenceGateway = { importPdfFromPicker: () => invoke('import_pdf_from_picker'), listRecentDocuments: () => invoke('list_recent_documents'), openDocument: (documentId: string) => invoke('open_document', { documentId }), - readDocumentBytes: (documentId: string) => invoke('read_document_bytes', { documentId }), - updatePageCount: (documentId: string, pageCount: number) => invoke('update_page_count', { documentId, pageCount }) + readDocumentBytes: (documentId: string, revisionId?: string) => invoke('read_document_bytes', { documentId, revisionId }), + updatePageCount: (revisionId: string, pageCount: number) => invoke('update_page_count', { revisionId, pageCount }), + extractPagesToDerivedRevision: (revisionId: string, startPage: number, endPage: number) => + invoke('extract_pages_to_derived_revision', { revisionId, startPage, endPage }), + triggerTextExtraction: (revisionId: string) => invoke('trigger_text_extraction', { revisionId }), + listExtractedPageText: (revisionId: string) => invoke('list_extracted_page_text', { revisionId }) }; diff --git a/docs/DESKTOP_FIRST_ARCHITECTURE.md b/docs/DESKTOP_FIRST_ARCHITECTURE.md index 548c0b7..93c3d76 100644 --- a/docs/DESKTOP_FIRST_ARCHITECTURE.md +++ b/docs/DESKTOP_FIRST_ARCHITECTURE.md @@ -1,341 +1,30 @@ -# Desktop-First Product Architecture Blueprint +# Desktop-First Product Architecture Blueprint (Updated) -## 1) Desktop-First Architecture Blueprint +This blueprint is updated to align with `docs/PDF_REVIEW_ARCHITECTURE_PACKAGE.md` and ADR-0002. -### Product direction -A local-first desktop application for engineering PDF review that treats **structured markup overlays** as the system of record and uses PDFs as immutable source artifacts. +## Core extension points now locked in -### Core architectural principles -1. **Desktop-native runtime**: Tauri shell is the only application runtime in normal use (no dependency on separate frontend/backend servers). -2. **Local-first data ownership**: SQLite + local filesystem are primary persistence layers. -3. **Offline-capable by default**: review, markup, comments, workflow state changes, and exports must work without network. -4. **Deterministic rendering + reversible edits**: PDF content is read-only; user intent lives in typed markup/comment/workflow entities. -5. **Traceability**: every meaningful state change emits an audit event. -6. **Modular boundaries from day one**: document pipeline, markup engine, workflow engine, persistence, and export service are isolated modules. +- Renderer abstraction layer (`viewer-core`) +- PDF.js adapter (`viewer-pdfjs`) +- Future native renderer adapter slot (same contract) +- Document transformation engine boundary +- OCR/indexing processing pipeline boundary +- Multi-layer render scene model +- Persistence for derived revisions, processing jobs, extracted page text, and audit events -### Runtime topology -- **Tauri Core (Rust)** - - App lifecycle and windowing - - Secure command surface (`invoke`) to frontend - - SQLite access (via `sqlx`/`rusqlite`) behind repository layer - - File I/O for PDF import/storage/export - - Export pipeline orchestration (PDF + overlays flattening) - - Audit event write-through -- **Frontend (React + TypeScript)** - - UI shells: Project, Document, Review Workspace, Workflow Inbox - - PDF.js viewer integration - - Overlay rendering layer for markups and comment anchors - - Local state (UI/session) + command/query client for Tauri APIs - - Validation and optimistic UI updates with rollback on persistence failure -- **Local storage** - - SQLite database: metadata, overlays, workflow objects, audit trail - - Filesystem store: imported source PDFs, generated export artifacts, thumbnails/cache +## Domain essentials -### Logical module decomposition -- `workspace`: project/document navigation and context loading -- `pdf_viewer`: PDF.js wrappers (page loading, viewport transforms, text layer hooks) -- `markup_engine`: shape models, hit-testing, coordinate transforms, editing tools -- `comments_engine`: threaded comments anchored to markup/page coordinates -- `workflow_engine`: reviewer assignment, squad checks, confirmations, reminders -- `persistence`: repositories and migrations for SQLite + file index -- `export_engine`: builds marked-up PDF outputs and review packages -- `audit_engine`: standardized event creation for all state transitions +- `Document` stable identity +- `DocumentRevision` immutable snapshots with `source_revision_id` and `derivation_type` +- `DocumentTransformationJob` (command + result, audit-linked) +- `ProcessingJob` (`text_extraction|ocr|thumbnail_generation|export`) +- `ExtractedPageText` page-addressable text content +- `ComparisonSession` base + optional overlay revision state +- `AuditEvent` for transformation/processing traceability -### Security and reliability baseline -- Tauri allowlist/capabilities locked to needed FS paths and commands only -- All writes in explicit transactions where entities are coupled -- Schema versioning + migrations at startup -- Crash-safe saves: append audit first, commit entity transaction, then fs sync where applicable -- Input validation for imported files and JSON payloads +## Architectural guardrails ---- - -## 2) Recommended Repo Structure (Tauri + React + SQLite + PDF.js) - -```text -GITPLANT/ - apps/ - desktop/ - src-tauri/ - Cargo.toml - tauri.conf.json - src/ - main.rs - app/ - mod.rs - commands/ - projects.rs - documents.rs - markups.rs - comments.rs - workflow.rs - export.rs - services/ - pdf_store.rs - export_service.rs - audit_service.rs - db/ - mod.rs - connection.rs - migrations.rs - repositories/ - projects_repo.rs - documents_repo.rs - markups_repo.rs - comments_repo.rs - workflow_repo.rs - audit_repo.rs - models/ - project.rs - document.rs - revision.rs - markup.rs - comment_thread.rs - reviewer_assignment.rs - squad_check.rs - confirmation.rs - reminder.rs - audit_event.rs - src/ - main.tsx - app/ - AppShell.tsx - routes.tsx - modules/ - workspace/ - viewer/ - PdfViewer.tsx - pdfjs.ts - viewport.ts - markup/ - OverlayCanvas.tsx - tools/ - serialization.ts - comments/ - workflow/ - export/ - state/ - queryClient.ts - stores/ - types/ - domain.ts - dto.ts - styles/ - packages/ - shared-types/ - src/ - domain.ts - events.ts - migrations/ - sqlite/ - 0001_init.sql - 0002_markup_workflow.sql - docs/ - DESKTOP_FIRST_ARCHITECTURE.md - MIGRATION_GUIDE_WEB_TO_DESKTOP.md -``` - -### Notes -- Frontend and Tauri Rust code are co-located under one desktop app package. -- No operational dependency on separate web backend process. -- Existing browser-first backend/frontend can be retained temporarily behind `legacy/` during migration. - ---- - -## 3) Domain Model - -### Project -- `id` (UUID) -- `name` (string) -- `description` (string?) -- `status` (`active|archived`) -- `created_at`, `updated_at` - -### Document -- `id` (UUID) -- `project_id` (FK) -- `title` (string) -- `discipline` (string?) -- `current_revision_id` (FK?) -- `created_at`, `updated_at` - -### DocumentRevision -- `id` (UUID) -- `document_id` (FK) -- `revision_label` (string, e.g., A, B, IFC-01) -- `source_pdf_path` (filesystem path) -- `page_count` (int) -- `checksum_sha256` (string) -- `imported_at` -- `is_superseded` (bool) - -### Markup -- `id` (UUID) -- `document_revision_id` (FK) -- `page_number` (int) -- `type` (`cloud|line|arrow|text|stamp|highlight|dimension|symbol`) -- `geometry` (JSON: normalized coordinates) -- `style` (JSON: color, thickness, opacity) -- `content` (JSON: optional text/payload) -- `status` (`open|resolved|void`) -- `created_by`, `created_at`, `updated_at` - -### CommentThread -- `id` (UUID) -- `document_revision_id` (FK) -- `markup_id` (FK nullable for page-level thread) -- `page_number` (int) -- `anchor` (JSON point/rect normalized) -- `state` (`open|resolved`) -- `created_by`, `created_at`, `updated_at` -- child `CommentMessage`: `id`, `thread_id`, `author`, `body`, `created_at`, `edited_at?` - -### ReviewerAssignment -- `id` (UUID) -- `project_id` (FK) -- `document_revision_id` (FK) -- `reviewer_id` (local user identity) -- `role` (`checker|approver|observer`) -- `due_at` (datetime?) -- `status` (`pending|in_progress|completed|reassigned`) -- `created_at`, `updated_at` - -### SquadCheck -- `id` (UUID) -- `document_revision_id` (FK) -- `check_type` (`discipline|constructability|safety|qa_custom`) -- `status` (`not_started|in_progress|passed|failed`) -- `owner_assignment_id` (FK) -- `started_at`, `completed_at` -- `result_summary` (text?) - -### Confirmation -- `id` (UUID) -- `target_type` (`markup|thread|squad_check|revision`) -- `target_id` (UUID) -- `confirmed_by` -- `confirmation_type` (`accepted|rejected|verified`) -- `note` (text?) -- `created_at` - -### Reminder -- `id` (UUID) -- `assignment_id` (FK nullable) -- `target_type` (`thread|squad_check|revision`) -- `target_id` (UUID) -- `scheduled_for` (datetime) -- `status` (`scheduled|sent|dismissed`) -- `channel` (`in_app|email_future`) -- `created_at`, `sent_at?` - -### AuditEvent -- `id` (UUID) -- `entity_type` (string) -- `entity_id` (UUID) -- `event_type` (string) -- `actor_id` (string) -- `timestamp` (datetime) -- `payload` (JSON diff/context) -- `revision` (monotonic integer or ULID for ordering) - ---- - -## 4) Data Flows - -### A) Opening a PDF -1. User selects file in desktop UI. -2. Frontend calls Tauri command `import_document_revision` with file path + project/document context. -3. Rust service validates extension/checksum, copies file into app-managed storage. -4. Rust stores `DocumentRevision` metadata in SQLite and emits `AuditEvent(document_revision.imported)`. -5. Frontend receives revision DTO and sets active review context. - -### B) Rendering a PDF page -1. Frontend PDF module opens local file via Tauri safe file API handle/path token. -2. PDF.js loads document and page bitmap/text layers. -3. Markup engine queries markups by `(document_revision_id, page_number)` from local DB. -4. Overlay canvas renders structured markups transformed to current viewport. -5. Comment anchors and workflow badges render as separate overlay layers. - -### C) Creating a markup -1. User chooses tool and draws shape on overlay canvas. -2. Frontend converts screen coordinates to normalized PDF coordinates. -3. Frontend builds `CreateMarkupInput` and invokes Tauri command. -4. Rust validates payload, inserts `Markup`, appends `AuditEvent(markup.created)` in transaction. -5. Frontend updates local state with persisted markup ID and timestamp. - -### D) Attaching comments -1. User opens/creates thread from markup or page anchor. -2. Frontend invokes `create_or_append_comment_thread`. -3. Rust upserts thread/message, updates thread state and audit trail transactionally. -4. Frontend refreshes thread panel and highlights related markup/thread anchor. - -### E) Saving to SQLite -1. Every mutating command is routed through Rust repositories. -2. Repositories enforce FK integrity, status transitions, and timestamp updates. -3. Command service wraps multi-entity writes in DB transaction. -4. On success, command returns full updated aggregate DTO. -5. On failure, no partial state; frontend rolls back optimistic state. - -### F) Exporting a marked-up PDF -1. User chooses export target and options (flatten overlays, include open comments, audit summary). -2. Frontend invokes `export_revision_package`. -3. Rust export service loads source PDF + markups/comments from SQLite. -4. Service applies overlay draw instructions per page and writes new PDF artifact. -5. Optional sidecar JSON/CSV includes workflow state and audit extracts. -6. Export metadata stored in DB and audit event emitted. - ---- - -## 5) Phased Implementation Plan - -### Phase 1 — Local Desktop MVP -- Bootstrap Tauri + React + TypeScript workspace. -- Integrate PDF.js viewer with page navigation and zoom. -- Implement local import/storage of PDFs and revision indexing. -- Implement core markup CRUD (cloud/arrow/text/highlight) and comment threads. -- Implement SQLite schema + migrations + audit event logging. -- Implement export of flattened marked-up PDF. - -**Exit criteria**: single-user local workflow from import → markup/comment → export with durable local persistence. - -### Phase 2 — Workflow Features -- Reviewer assignments and per-revision inbox. -- Squad check templates/status tracking. -- Confirmations and reminders with in-app notification center. -- Rich filtering/search over markups, threads, due items. -- Improved audit trail views and timeline. - -**Exit criteria**: end-to-end desktop workflow management for engineering review cycles. - -### Phase 3 — Sync/Collaboration -- Optional sync adapter (cloud or on-prem) with conflict-aware merge on overlay objects. -- Identity, roles, and encrypted sync transport. -- Shared review state, assignment updates, and audit replication. -- Background sync and offline reconciliation UX. - -**Exit criteria**: multi-user collaboration while preserving local-first resilience. - ---- - -## 6) Proposed Refactor Plan for Current Repo - -### Current state (high-level) -Repository is organized as browser-first split frontend/backend with FastAPI + Vite and dev server orchestration. - -### Refactor strategy -1. **Create new desktop app root**: add `apps/desktop` (Tauri + React), keep current stack as `legacy/` during migration. -2. **Extract domain contracts**: define shared TS/Rust DTO contracts for Project/DocumentRevision/Markup/etc. -3. **Re-home persistence**: move data models from HTTP backend semantics to SQLite repositories in Tauri Rust. -4. **Port UI feature slices**: migrate project/document views first, then replace browser API client with Tauri command client. -5. **Replace backend endpoints with commands**: each prior API capability becomes typed Tauri command + service layer. -6. **Introduce PDF.js workspace**: add review workspace route with overlay engine as first-class module. -7. **Add workflow modules**: assignments, squad check, confirmations, reminders, audit timeline. -8. **Sunset legacy runtime**: remove dependency on separate backend/frontend servers for production paths. - -### Suggested execution sequence for this repo -- Step A: Add architecture + migration docs and agreed domain schema (this document). -- Step B: Scaffold Tauri desktop app with minimal shell and compile checks. -- Step C: Implement SQLite + migrations + project/document/revision CRUD. -- Step D: Integrate PDF.js + markup/comment flows. -- Step E: Add export engine and workflow modules. -- Step F: Decommission legacy server paths after parity checks. +1. Viewer/UI code must not directly import PDF.js internals. +2. Transform operations must create new revisions (no in-place mutation). +3. OCR and text extraction are processing concerns, not rendering concerns. +4. Originals remain immutable in managed local storage. diff --git a/docs/PDF_REVIEW_ARCHITECTURE_PACKAGE.md b/docs/PDF_REVIEW_ARCHITECTURE_PACKAGE.md index 46d518b..a8d6c7f 100644 --- a/docs/PDF_REVIEW_ARCHITECTURE_PACKAGE.md +++ b/docs/PDF_REVIEW_ARCHITECTURE_PACKAGE.md @@ -1,591 +1,83 @@ # Desktop-First Engineering PDF Review — Architecture Package -## 1. Executive summary - -This product pivot should be implemented as a **desktop-first, local-first engineering PDF review system** with a strict separation between rendering infrastructure and product logic. - -- **Desktop runtime**: Tauri shell + React/TypeScript UI + Vite. -- **Rendering v1**: PDF.js adapter behind a renderer interface. -- **Core product engine**: markups, comments, reviewer workflow, confirmations, reminders, audit trail. -- **Persistence**: SQLite for structured data; local filesystem for source PDFs and generated outputs. -- **Performance**: viewport/tile rendering, multi-resolution caches, prefetching, memory budgeting, and overlay independence. -- **Future-proofing**: a renderer contract that allows a native sidecar (PDFium-class) later without rewriting workflow/markup/comment logic. - -This architecture is optimized for Oil & Gas engineering review of large PDFs where reliability, traceability, and desktop productivity matter more than browser-native convenience. - ---- - -## 2. Architecture blueprint - -### 2.1 Desktop application overview - -**Application topology** -- **Tauri shell (Rust)** - - Window lifecycle, secure capabilities, native menus/shortcuts - - SQLite access and migrations - - File-system orchestration for PDF import/export and workspace paths - - Optional sidecar process management (future native renderer) -- **React + TypeScript frontend (Vite)** - - Viewer workspace UI and tool system - - Overlay rendering for markups/comments - - Workflow boards and reviewer assignment UI - - Event-driven state store for active document sessions -- **Local persistence** - - SQLite as system-of-record for structured entities - - Filesystem object store for immutable PDFs and export artifacts - - Disk caches for tiles/thumbnails/preprocessed page assets - -### 2.2 Major subsystems - -1. **Workspace & Project Context** - - Project selection, document list, revision history, current review state. -2. **Viewer Core** - - Viewport control, page navigation, zoom/pan, hit-testing bridge. -3. **Renderer Abstraction Layer** - - Stable interface consumed by Viewer Core. - - Adapter implementation for PDF.js in v1. -4. **Markup Core** - - Geometry, anchors, style models, edit operations, status transitions. -5. **Comment Core** - - Thread lifecycle and linkage to markup/page anchors. -6. **Workflow Core** - - Reviewer assignment, squad check, confirmations, reminders. -7. **Persistence Layer** - - Repositories, unit-of-work transactions, migrations. -8. **Export Engine** - - Deterministic flattening of overlays/comments/workflow summary into derived outputs. -9. **Audit Engine** - - Immutable event capture of meaningful actions. - -### 2.3 Renderer abstraction strategy - -- The app never calls PDF.js APIs directly outside `viewer-pdfjs` package. -- Viewer consumes only `RendererProvider` interface and typed DTOs. -- Markup/Workflow/Persistence layers cannot import renderer implementation modules. -- Coordinate systems standardized in shared contracts (page space as normalized or document units, screen space as viewport pixels). - -### 2.4 Local-first data architecture - -**Data ownership rules** -- **Source of truth for review work**: SQLite rows for markups/comments/workflow/audit. -- **Source PDFs**: immutable files in workspace-managed storage. -- **Derived files**: exports/previews are reproducible outputs, never authoritative. - -**Durability model** -- Writes grouped transactionally by action boundary (e.g., markup create + audit event). -- File import uses checksum and write-then-rename atomic pattern. -- Crash recovery through transaction rollback + startup integrity checks. - -### 2.5 Performance strategy for large PDFs - -- Virtualized page list; render only visible/near-visible pages. -- Tile-based rendering at high zoom to avoid full-page reraster costs. -- Multi-level caches (in-memory + disk-backed) keyed by document revision, page, zoom bucket, tile. -- Overlay redraw decoupled from PDF raster refresh. -- Aggressive cancellation/debouncing of stale render jobs during pan/zoom. - -### 2.6 Export strategy - -- Export pipeline reads: - 1) source PDF revision, - 2) filtered markup/comment/workflow state snapshot, - 3) export profile (flattening rules, included layers, stamp template). -- Output targets: - - Flattened review PDF - - Optional companion JSON package (structured review data) - - Optional audit summary report -- Exports are **versioned jobs** with reproducible inputs and checksums. - -### 2.7 Workflow engine boundaries - -Workflow core governs: -- assignment and ownership, -- squad check progression, -- confirmations/reminders, -- status filtering and SLA-like due visibility. - -Workflow core does **not**: -- perform PDF rendering, -- own geometry/hit-testing, -- write directly to files. - -### 2.8 Future sync/collaboration expansion path - -Design now for later optional sync: -- Local event log and change stamps on entities. -- Conflict-safe identifiers and monotonic version counters. -- Sync adapter boundary (push/pull entity deltas + file manifests) without coupling local UX to network availability. -- Collaboration remains additive; desktop-local operation remains primary mode. - -### 2.9 Deployment/distribution model for desktop - -- Build signed installers per OS target (MSI/EXE for Windows first, optional macOS/Linux later). -- Auto-update channel with staged rollout rings. -- Workspace storage path configurable by policy. -- Enterprise-friendly offline installer option. - -### 2.10 Why this architecture is right for large engineering PDFs - -- Heavy documents require native-like local I/O, memory control, and responsive view interaction. -- Engineering review value lies in markup/workflow traceability, not raw rendering APIs. -- Renderer abstraction protects long-term performance options while preserving domain logic investments. -- Local-first guarantees dependable site/offline usage common in project environments. - ---- - -## 3. Renderer abstraction design - -### 3.1 Core interface (contract) - -```ts -export interface RendererProvider { - openDocument(input: OpenDocumentInput): Promise; - closeDocument(documentId: string): Promise; - getDocumentMetadata(documentId: string): Promise; - getPageCount(documentId: string): Promise; - getPageInfo(documentId: string, pageNumber: number): Promise; - renderViewport( - documentId: string, - pageNumber: number, - viewport: ViewportSpec, - options?: RenderOptions - ): Promise; - renderTile( - documentId: string, - pageNumber: number, - tile: TileSpec, - options?: RenderOptions - ): Promise; - getTextContent(documentId: string, pageNumber: number): Promise; - screenToPage( - pageNumber: number, - x: number, - y: number, - viewportState: ViewportState - ): PagePoint; - pageToScreen( - pageNumber: number, - x: number, - y: number, - viewportState: ViewportState - ): ScreenPoint; -} -``` - -### 3.2 Adapter responsibilities - -The renderer adapter **owns**: -- Loading/parsing document bytes through underlying renderer. -- Page raster and text extraction details. -- Conversion between renderer-native coordinate system and contract coordinate model. -- Render job lifecycle/cancellation and low-level caching primitives. - -### 3.3 Renderer-agnostic responsibilities - -The following must remain independent of PDF.js: -- Markup object schema and editing operations. -- Comment thread anchoring model. -- Workflow states and transitions. -- Persistence repositories and audit event generation. -- Export business rules (which entities included, sign-off metadata, status filters). - -### 3.4 Swap strategy for native renderer later - -To replace PDF.js: -1. Implement `RendererProvider` in `viewer-native` package (Tauri sidecar or Rust binding). -2. Preserve existing shared DTO contracts. -3. Keep Viewer Core integration unchanged except provider binding configuration. -4. Run renderer compatibility suite (metadata parity, coordinate transform parity, tile fidelity thresholds). - -This keeps replacement scope contained to adapter package + performance tuning, not cross-application rewrite. - ---- - -## 4. Performance strategy for large PDFs - -### 4.1 Viewport-based rendering - -- Use scroll virtualization; only instantiate page controllers for visible range + buffer. -- Trigger raster generation from viewport observer events. -- Cancel renders when page exits active range. - -### 4.2 Tile rendering strategy - -- At low/medium zoom: single full-page bitmap acceptable. -- At high zoom: split page into tiles (e.g., 512x512 or 1024x1024 logical tiles). -- Prioritize center-of-viewport tiles first; defer edge tiles. - -### 4.3 Multi-resolution rendering - -- Maintain zoom buckets (e.g., 0.5x, 1x, 2x, 4x equivalent DPI targets). -- Show nearest cached resolution immediately, then refine progressively. -- Prevent blocking UI on exact-resolution render completion. - -### 4.4 Page/tile cache strategy - -- **L1 memory cache**: recent tiles/bitmaps with weighted LRU eviction. -- **L2 disk cache**: optional persisted tile blobs keyed by checksum + render params. -- Include page rotation, crop/view mode, and color profile in cache key. - -### 4.5 Lazy loading strategy - -- Delay heavy metadata/text extraction until page is requested. -- Load text layer on demand (search/select mode), not unconditionally. -- Defer thumbnail generation for unopened pages. - -### 4.6 Overlay redraw independence - -- Markup/comment overlays rendered on independent canvas/SVG layers. -- During pan/zoom, transform overlay matrix immediately while PDF tiles refresh asynchronously. -- Editing interactions use vector model in page coordinates; no raster dependency for drag/resize feedback. - -### 4.7 Memory management approach - -- Global memory budget (configurable) for raster caches. -- Pressure signals trigger eviction by distance from viewport and stale zoom buckets. -- Reuse canvas buffers where possible to reduce allocations and GC churn. - -### 4.8 Prefetch strategy for nearby pages - -- Prefetch next/previous pages based on scroll direction and speed. -- Prefetch limited tile ring around current viewport at likely next zoom bucket. -- Stop prefetch under active editing bursts or memory pressure. - -### 4.9 Bottlenecks and mitigations (engineering PDFs) - -1. **Very large vector-heavy pages** - - Mitigation: tile render + job cancellation + progressive detail. -2. **Frequent zoom/pan during redline review** - - Mitigation: debounced rerender, matrix-transformed overlays, stale job pruning. -3. **Large markups per page** - - Mitigation: spatial index for hit-testing and partial overlay redraw. -4. **High page-count documents (hundreds/thousands)** - - Mitigation: strict page virtualization and metadata lazy load. - ---- - -## 5. Domain model - -### 5.1 Workspace/Project -- **Purpose**: logical container for documents, users, and review workflow scope. -- **Key fields**: `id`, `name`, `code`, `facility`, `status`, `createdAt`, `updatedAt`. -- **Relationships**: one-to-many with `Document`, `ReviewerAssignment`, `Reminder`. -- **Lifecycle**: `active -> archived`. - -### 5.2 Document -- **Purpose**: stable identity for an engineering deliverable across revisions. -- **Key fields**: `id`, `workspaceId`, `documentNumber`, `title`, `discipline`, `currentRevisionId`, `status`. -- **Relationships**: one-to-many with `DocumentRevision`; one current revision pointer. -- **Lifecycle**: `drafted -> under_review -> accepted -> superseded`. - -### 5.3 DocumentRevision -- **Purpose**: immutable imported PDF revision and associated review context. -- **Key fields**: `id`, `documentId`, `revisionLabel`, `filePath`, `checksumSha256`, `pageCount`, `importedAt`, `isCurrent`. -- **Relationships**: one-to-many with `MarkupLayer`, `ReviewerAssignment`, `SquadCheck`, `ExportJob`. -- **Lifecycle**: `imported -> active_review -> closed`. - -### 5.4 MarkupLayer -- **Purpose**: logical grouping/filtering boundary for markup visibility and export. -- **Key fields**: `id`, `revisionId`, `name`, `kind` (discipline/reviewer/system), `visibility`, `locked`. -- **Relationships**: one-to-many with `MarkupObject`. -- **Lifecycle**: `active -> locked -> archived`. - -### 5.5 MarkupObject -- **Purpose**: atomic redline/annotation element on a page. -- **Key fields**: `id`, `layerId`, `pageNumber`, `type`, `geometryJson`, `styleJson`, `status`, `createdBy`, `updatedBy`, `createdAt`, `updatedAt`. -- **Relationships**: optional one-to-one or one-to-many with `CommentThread`; linked audit events. -- **Lifecycle**: `open -> resolved -> reopened` (or `void` for invalidated markups). - -### 5.6 CommentThread -- **Purpose**: conversation anchored to markup or page region. -- **Key fields**: `id`, `revisionId`, `markupObjectId?`, `pageNumber`, `anchorJson`, `state`, `createdBy`, `createdAt`, `updatedAt`. -- **Relationships**: one-to-many with `Comment`. -- **Lifecycle**: `open -> resolved -> reopened`. - -### 5.7 Comment -- **Purpose**: message item within a thread. -- **Key fields**: `id`, `threadId`, `authorId`, `body`, `mentionsJson`, `createdAt`, `editedAt?`, `deletedAt?`. -- **Relationships**: belongs to `CommentThread`. -- **Lifecycle**: `active -> edited -> deleted` (soft-delete for auditability). - -### 5.8 ReviewerAssignment -- **Purpose**: assigns review responsibility to a person/role for revision scope. -- **Key fields**: `id`, `workspaceId`, `revisionId`, `reviewerId`, `role`, `dueAt`, `status`, `assignedBy`, `assignedAt`. -- **Relationships**: can own `SquadCheck` and be referenced by reminders. -- **Lifecycle**: `pending -> in_progress -> completed` or `reassigned`. - -### 5.9 SquadCheck -- **Purpose**: structured multidisciplinary check activity tied to revision. -- **Key fields**: `id`, `revisionId`, `checkType`, `ownerAssignmentId`, `status`, `result`, `startedAt`, `completedAt`. -- **Relationships**: may require `Confirmation` records before pass state. -- **Lifecycle**: `not_started -> in_progress -> confirmed_pass|confirmed_fail`. - -### 5.10 Confirmation -- **Purpose**: explicit acknowledgment/acceptance against workflow targets. -- **Key fields**: `id`, `targetType`, `targetId`, `confirmedBy`, `decision`, `note`, `createdAt`. -- **Relationships**: references `MarkupObject`, `CommentThread`, or `SquadCheck` targets. -- **Lifecycle**: append-only decision records (no hard overwrite). - -### 5.11 Reminder -- **Purpose**: time-based nudges for pending assignments/checks/resolutions. -- **Key fields**: `id`, `workspaceId`, `targetType`, `targetId`, `recipientId`, `dueAt`, `sentAt?`, `status`. -- **Relationships**: linked to assignments/checks/threads. -- **Lifecycle**: `scheduled -> sent -> acknowledged|expired`. - -### 5.12 AuditEvent -- **Purpose**: immutable compliance trace of state-changing actions. -- **Key fields**: `id`, `workspaceId`, `entityType`, `entityId`, `action`, `actorId`, `timestamp`, `payloadJson`, `hash`. -- **Relationships**: references any domain entity. -- **Lifecycle**: append-only. - -### 5.13 ExportJob -- **Purpose**: tracks deterministic generation of flattened outputs. -- **Key fields**: `id`, `revisionId`, `profile`, `status`, `requestedBy`, `requestedAt`, `completedAt?`, `outputPath?`, `inputSnapshotHash`. -- **Relationships**: belongs to `DocumentRevision`; generates derived files. -- **Lifecycle**: `queued -> running -> succeeded|failed|cancelled`. - ---- - -## 6. Data flow design - -### 6.1 Import/open a PDF -1. User selects file in desktop UI. -2. Tauri command copies file to managed workspace path, computes checksum. -3. Renderer opens file for metadata/page count extraction. -4. Persistence transaction creates/updates `Document` + inserts `DocumentRevision`. -5. Audit event appended (`document_revision_imported`). - -**Storage placement** -- SQLite: document + revision metadata + audit. -- Local files: source PDF. -- In-memory: opened renderer handle and initial page cache entries. - -### 6.2 Render a page -1. Viewer requests page render from `RendererProvider` with viewport/tile spec. -2. Adapter serves from cache or schedules raster job. -3. Rendered surface returned to viewer and painted. -4. Overlay engine draws markups/comments from in-memory revision snapshot. - -**Storage placement** -- In-memory caches: raster tiles, viewport state. -- Renderer layer: document/page handles. -- SQLite: no write path for pure render. - -### 6.3 Create a markup -1. User draws object in overlay layer (page coordinates). -2. Markup core validates geometry and snaps to model constraints. -3. Transaction inserts `MarkupObject` and `AuditEvent`. -4. UI updates overlay state and index structures. - -**Storage placement** -- SQLite: markup row + audit event. -- In-memory: overlay scene graph and spatial index. - -### 6.4 Update a markup -1. User edits geometry/style/status. -2. Markup core computes patch + validation. -3. Transaction updates `MarkupObject` and inserts `AuditEvent`. -4. Overlay redraws changed object only. - -### 6.5 Attach comment thread to markup -1. User opens comment panel on markup. -2. Transaction creates `CommentThread`, first `Comment`, and `AuditEvent`. -3. UI links thread badge to markup anchor. - -**Storage placement** -- SQLite: thread/comment/audit. -- In-memory: thread panel cache. - -### 6.6 Resolve/reopen markup issue -1. Status change requested from markup or thread context. -2. Domain rule check (e.g., unresolved mandatory confirmations may block resolve). -3. Transaction updates status fields and appends audit entry. -4. Workflow filters refresh counts and lists. - -### 6.7 Assign reviewer -1. Workflow UI creates assignment request. -2. Transaction inserts `ReviewerAssignment` + optional `Reminder` seed + `AuditEvent`. -3. Work queue view refreshes for assignee. - -### 6.8 Confirm squad check -1. Reviewer submits check result and confirmation decision. -2. Transaction updates `SquadCheck`, inserts `Confirmation`, and `AuditEvent`. -3. Dependent workflow states recomputed (e.g., revision review gate conditions). - -### 6.9 Export marked-up PDF -1. User creates `ExportJob` with profile. -2. Export engine snapshots relevant entities from SQLite. -3. Renderer loads source revision PDF and applies flattening instructions. -4. Output file written to exports directory; job status updated. -5. Audit event appended. - -**Storage placement** -- SQLite: export job status + audit. -- Local files: generated PDF and optional JSON/report. -- Renderer layer: transient render/write context. -- Future sync service: publish export manifest and hashes only when enabled. - ---- - -## 7. Repo refactor plan - -### 7.1 Target structure - -```text -GITPLANT/ - apps/ - desktop/ - src/ - app/ - features/ - workspace/ - viewer/ - markup/ - comments/ - workflow/ - infrastructure/ - renderer-client/ - persistence-client/ - src-tauri/ - src/ - commands/ - services/ - db/ - fs/ - export/ - packages/ - shared-types/ - src/ - domain/ - renderer/ - workflow/ - audit/ - viewer-core/ - src/ - contracts/ - viewport/ - cache/ - controller/ - viewer-pdfjs/ - src/ - PdfjsRendererProvider.ts - transforms/ - text/ - markup-core/ - src/ - models/ - tools/ - rules/ - workflow-core/ - src/ - models/ - transitions/ - services/ - persistence-sqlite/ - src/ - repositories/ - migrations/ - unit-of-work/ - export-core/ - src/ - pipeline/ - profiles/ - flattening/ - docs/ - PDF_REVIEW_ARCHITECTURE_PACKAGE.md - MIGRATION_PLAN_DESKTOP_FIRST.md -``` - -### 7.2 Why this structure supports growth and renderer swap - -- `viewer-core` depends only on renderer contracts, never PDF.js. -- `viewer-pdfjs` is replaceable by `viewer-native` with parallel contract tests. -- `markup-core` and `workflow-core` are independently testable and reusable. -- Tauri backend concerns (SQLite/file/export/security) remain isolated from UI. -- Shared types prevent drift between Rust commands, TS UI, and package modules. - ---- - -## 8. Phased implementation roadmap - -### Phase 1: Desktop foundation + review core - -**Goals** -- Tauri desktop shell operational. -- PDF open/view through renderer abstraction. -- SQLite schema + repositories. -- Markup overlay model. -- Comment threads. -- Audit log append for state changes. - -**Dependencies** -- Baseline repo refactor complete. -- Shared contracts finalized. -- Initial migration set for core entities. - -**Done criteria** -- Open large PDF locally and navigate pages smoothly. -- Create/update/resolve markup and comment thread with persisted state. -- Restart app and recover full local review state. -- Audit events generated for core operations. - -**Risks** -- Early coupling to PDF.js internals in UI. -- Unbounded memory from naive raster caching. -- SQLite transaction boundaries not aligned to domain operations. - -### Phase 2: Workflow execution layer - -**Goals** -- Reviewer assignment system. -- Squad check flow. -- Confirmations and reminders. -- Status filtering and dashboard views. - -**Dependencies** -- Stable identity/user model for local actors. -- Workflow transition rules in core package. - -**Done criteria** -- Assign reviewers and track completion state. -- Execute squad checks with confirmations and audit trace. -- Reminder scheduling and acknowledgement lifecycle functional. - -**Risks** -- Workflow complexity growth without explicit state machine tests. -- Reminder delivery UX unclear in offline-only sessions. - -### Phase 3: Export + performance hardening + collaboration path - -**Goals** -- Export engine for flattened PDFs and structured review bundles. -- Advanced performance tuning (tile cache policy, prefetch heuristics). -- Sync/collaboration architecture scaffolding. -- Optional native renderer proof-of-path. - -**Dependencies** -- Stable domain schema and workflow states. -- Renderer contract test suite. - -**Done criteria** -- Deterministic exports with traceable job records. -- Measured performance targets met on representative large engineering PDFs. -- Sync interfaces defined and local event log suitable for replication. -- Native renderer feasibility validated without app-wide refactor. - -**Risks** -- Export fidelity differences across renderer implementations. -- Cross-platform packaging/signing constraints. -- Scope creep into real-time collaboration before local robustness. - ---- - -## 9. Recommended immediate next coding step - -**Next step**: implement the **contract-only foundation** (no feature expansion yet): -1. Create `packages/shared-types` with renderer/domain/workflow type contracts. -2. Create `packages/viewer-core` with `RendererProvider` interface and contract tests. -3. Create `packages/viewer-pdfjs` minimal adapter stub implementing the interface for open/count/render metadata smoke tests. -4. Add architecture decision records (ADRs) for local-first data authority and renderer swappability. - -This is the smallest coding slice that locks architectural boundaries before feature implementation. +## Executive summary +The desktop app remains local-first (Tauri + React + SQLite + managed filesystem), but architecture is now explicitly upgraded for: +- multi-layer page rendering/comparison, +- immutable document transformations with derived revisions, +- OCR/indexing processing pipelines with per-page text persistence. + +## Updated architecture blueprint + +### Subsystems and boundaries +- **Renderer abstraction (`packages/viewer-core`)** + - Owns `RenderScene`, `RenderLayer`, viewport contracts, renderer interface. +- **PDF.js adapter (`packages/viewer-pdfjs`)** + - Implements renderer contract and text-content hooks for searchable PDFs. +- **Future native renderer adapter slot** + - Kept behind the same viewer-core interface; UI stays adapter-agnostic. +- **Document transformation engine (`packages/document-transform-core` + Tauri Rust service)** + - Owns transformation commands/results and derived-revision lifecycle. +- **OCR/indexing processing pipeline (`packages/processing-core` + `packages/text-extraction-core` + Tauri Rust service)** + - Owns processing jobs, status transitions, and extracted text storage. +- **Persistence core + SQLite (`packages/persistence-core`, `apps/desktop/src-tauri`)** + - Owns revision lineage, jobs, extracted text, and audit events. +- **Desktop UI (`apps/desktop/src`)** + - Uses abstractions only (viewer-core/persistence contracts), not PDF.js internals. + +## Domain model update + +### Document +Stable identity for a managed engineering document. + +### DocumentRevision +- immutable file snapshot +- `source_revision_id` nullable for lineage +- `derivation_type` nullable for transformation provenance + +### PageAsset / PageRenderCache metadata (scaffold) +- optional cache metadata keyed by revision/page/render params + +### DocumentTransformationJob +- command intent + execution status + output revision + +### ProcessingJob +- `job_type`: `text_extraction | ocr | thumbnail_generation | export` +- `status`: `pending | running | completed | failed` + +### ExtractedPageText +- page-addressable extracted/search text stored independently from raster output + +### ComparisonSession +- viewer state linking base revision + optional overlay revision + current page/viewport + +### AuditEvent +- immutable records for transformation and processing events + +## Multi-layer viewer model +Viewer scene is layer-based, not single-raster based: +- `LayerKind`: `base_pdf`, `overlay_pdf`, `markup_overlay`, `selection_overlay` +- per-layer visibility, opacity, transform placeholders +- supports base-only scene today and optional second PDF layer for comparison proof + +## Persistence model (SQLite) +Key entities: +- `documents` +- `document_revisions` (lineage + derivation) +- `processing_jobs` +- `extracted_page_text` +- `audit_events` +- `recent_documents` + +## Implemented now vs scaffolded + +### Implemented now +- base + optional overlay layer scene modeling in viewer-core/UI +- one real transformation proof path: extract page range into derived revision +- real text extraction job path for searchable PDFs with per-page text persistence +- audit events for transformation + processing completions + +### Scaffolded for later +- full OCR provider implementation +- full UI for all transform operations +- markup/selection overlay rendering engines (layer slots are ready) +- native renderer adapter implementation diff --git a/docs/adr/0002-multi-layer-transform-processing-boundaries.md b/docs/adr/0002-multi-layer-transform-processing-boundaries.md new file mode 100644 index 0000000..4d02927 --- /dev/null +++ b/docs/adr/0002-multi-layer-transform-processing-boundaries.md @@ -0,0 +1,29 @@ +# ADR 0002: Multi-layer viewer + transformation engine + processing pipeline boundaries + +## Status +Accepted + +## Context +The desktop-first baseline already separates renderer infrastructure from product-domain behavior, but Pass 2 requires explicit support for derived revisions, processing jobs, and comparison-ready page layering before deeper features are implemented. + +## Decision +1. **PDF rendering stays infrastructure** + - Rendering remains behind `viewer-core` contracts. + - `viewer-pdfjs` is an adapter; future native renderer is another adapter slot. +2. **Markup/workflow stay product-core** + - Markup, comments, reviewer workflow, confirmations, and audit remain in domain/persistence services, not renderer code. +3. **Document transformations are a dedicated engine** + - Transform operations are modeled as commands (`delete_pages`, `insert_pages`, `reorder_pages`, `extract_pages`, `combine_documents`). + - Viewer is read-only and never mutates source files. +4. **OCR/indexing is a processing pipeline** + - Processing jobs are tracked independently from rendering. + - `text_extraction` and `ocr` are distinct job types. +5. **Viewer supports stacked layers from day one** + - Render scenes include multiple `RenderLayer` records (`base_pdf`, `overlay_pdf`, future markup/selection layers). +6. **Originals are immutable** + - Imported revisions are immutable. + - Transformations create derived `DocumentRevision` rows with `source_revision_id`. + +## Consequences +- Existing Pass 1 flow still works while enabling comparison, text search, and transform jobs without architecture rewrites. +- SQLite schema now stores revision lineage, processing jobs, extracted page text, and audit events for transform/processing actions. diff --git a/packages/document-transform-core/package.json b/packages/document-transform-core/package.json new file mode 100644 index 0000000..1a5347c --- /dev/null +++ b/packages/document-transform-core/package.json @@ -0,0 +1,6 @@ +{ + "name": "@gitplant/document-transform-core", + "version": "0.1.0", + "type": "module", + "main": "src/index.ts" +} diff --git a/packages/document-transform-core/src/index.ts b/packages/document-transform-core/src/index.ts new file mode 100644 index 0000000..196d034 --- /dev/null +++ b/packages/document-transform-core/src/index.ts @@ -0,0 +1,24 @@ +export type TransformationKind = + | 'delete_pages' + | 'insert_pages' + | 'reorder_pages' + | 'extract_pages' + | 'combine_documents'; + +export interface TransformationRequest { + kind: TransformationKind; + sourceRevisionId: string; + payload: Record; +} + +export interface TransformationResult { + documentId: string; + sourceRevisionId: string; + derivedRevisionId: string; + managedFilePath: string; + pageCount: number; +} + +export interface DocumentTransformEngine { + run(request: TransformationRequest): Promise; +} diff --git a/packages/document-transform-pdflib/package.json b/packages/document-transform-pdflib/package.json new file mode 100644 index 0000000..4d967ec --- /dev/null +++ b/packages/document-transform-pdflib/package.json @@ -0,0 +1,9 @@ +{ + "name": "@gitplant/document-transform-pdflib", + "version": "0.1.0", + "type": "module", + "main": "src/index.ts", + "dependencies": { + "@gitplant/document-transform-core": "0.1.0" + } +} diff --git a/packages/document-transform-pdflib/src/index.ts b/packages/document-transform-pdflib/src/index.ts new file mode 100644 index 0000000..7cc95b7 --- /dev/null +++ b/packages/document-transform-pdflib/src/index.ts @@ -0,0 +1,7 @@ +import type { DocumentTransformEngine, TransformationRequest, TransformationResult } from '@gitplant/document-transform-core'; + +export class PdfLibTransformAdapter implements DocumentTransformEngine { + async run(_request: TransformationRequest): Promise { + throw new Error('Transformation adapter is implemented in Tauri Rust for desktop-managed storage.'); + } +} diff --git a/packages/persistence-core/src/index.ts b/packages/persistence-core/src/index.ts index 58bab92..02437d2 100644 --- a/packages/persistence-core/src/index.ts +++ b/packages/persistence-core/src/index.ts @@ -1,16 +1,32 @@ -import type { RecentDocumentView } from '@gitplant/shared-types'; +import type { + ProcessingJobRecord, + ExtractedPageTextRecord, + RecentDocumentView +} from '@gitplant/shared-types'; export interface ImportedDocumentPayload { documentId: string; title: string; managedFilePath: string; fileSizeBytes: number; + revisionId: string; +} + +export interface DerivedRevisionPayload { + documentId: string; + sourceRevisionId: string; + derivedRevisionId: string; + managedFilePath: string; + pageCount: number; } export interface PersistenceGateway { importPdfFromPicker(): Promise; listRecentDocuments(): Promise; openDocument(documentId: string): Promise; - readDocumentBytes(documentId: string): Promise; - updatePageCount(documentId: string, pageCount: number): Promise; + readDocumentBytes(documentId: string, revisionId?: string): Promise; + updatePageCount(revisionId: string, pageCount: number): Promise; + extractPagesToDerivedRevision(revisionId: string, startPage: number, endPage: number): Promise; + triggerTextExtraction(revisionId: string): Promise; + listExtractedPageText(revisionId: string): Promise; } diff --git a/packages/processing-core/package.json b/packages/processing-core/package.json new file mode 100644 index 0000000..37d1d8e --- /dev/null +++ b/packages/processing-core/package.json @@ -0,0 +1,9 @@ +{ + "name": "@gitplant/processing-core", + "version": "0.1.0", + "type": "module", + "main": "src/index.ts", + "dependencies": { + "@gitplant/shared-types": "0.1.0" + } +} diff --git a/packages/processing-core/src/index.ts b/packages/processing-core/src/index.ts new file mode 100644 index 0000000..d48a41e --- /dev/null +++ b/packages/processing-core/src/index.ts @@ -0,0 +1,21 @@ +import type { ProcessingJobStatus, ProcessingJobType } from '@gitplant/shared-types'; + +export interface ProcessingJobRequest { + revisionId: string; + jobType: ProcessingJobType; + payload?: Record; +} + +export interface ProcessingProvider { + supports(jobType: ProcessingJobType): boolean; + run(jobId: string, request: ProcessingJobRequest): Promise; +} + +export interface OcrProvider { + extractFromImagePdf(_revisionId: string): Promise; +} + +export interface ProcessingJobState { + id: string; + status: ProcessingJobStatus; +} diff --git a/packages/shared-types/src/index.ts b/packages/shared-types/src/index.ts index deaf56f..dfb0c52 100644 --- a/packages/shared-types/src/index.ts +++ b/packages/shared-types/src/index.ts @@ -5,16 +5,24 @@ export interface DocumentRecord { updatedAt: string; } +export type DerivationType = + | 'imported_original' + | 'extract_pages' + | 'combine_documents' + | 'delete_pages' + | 'insert_pages' + | 'reorder_pages'; + export interface DocumentRevisionRecord { id: string; documentId: string; - revisionNumber: number; managedFilePath: string; originalFileName: string; - sourcePath?: string | null; pageCount?: number | null; fileSizeBytes: number; importedAt: string; + sourceRevisionId?: string | null; + derivationType?: DerivationType | null; } export interface RecentDocumentRecord { @@ -29,4 +37,43 @@ export interface RecentDocumentView { managedFilePath: string; openedAt: string; pageCount?: number | null; + activeRevisionId?: string; +} + +export type ProcessingJobType = 'text_extraction' | 'ocr' | 'thumbnail_generation' | 'export'; +export type ProcessingJobStatus = 'pending' | 'running' | 'completed' | 'failed'; + +export interface ProcessingJobRecord { + id: string; + revisionId: string; + jobType: ProcessingJobType; + status: ProcessingJobStatus; + payloadJson?: string | null; + errorMessage?: string | null; + createdAt: string; + startedAt?: string | null; + completedAt?: string | null; +} + +export interface ExtractedPageTextRecord { + id: string; + revisionId: string; + pageNumber: number; + textContent: string; + extractedAt: string; +} + +export interface ComparisonSession { + baseRevisionId: string; + overlayRevisionId?: string; + pageNumber: number; +} + +export interface AuditEvent { + id: string; + entityType: string; + entityId: string; + eventType: string; + payloadJson?: string | null; + occurredAt: string; } diff --git a/packages/text-extraction-core/package.json b/packages/text-extraction-core/package.json new file mode 100644 index 0000000..5ca2115 --- /dev/null +++ b/packages/text-extraction-core/package.json @@ -0,0 +1,6 @@ +{ + "name": "@gitplant/text-extraction-core", + "version": "0.1.0", + "type": "module", + "main": "src/index.ts" +} diff --git a/packages/text-extraction-core/src/index.ts b/packages/text-extraction-core/src/index.ts new file mode 100644 index 0000000..7db0411 --- /dev/null +++ b/packages/text-extraction-core/src/index.ts @@ -0,0 +1,8 @@ +export interface PageText { + pageNumber: number; + text: string; +} + +export interface TextExtractionProvider { + extractPageText(revisionId: string): Promise; +} diff --git a/packages/viewer-core/src/index.ts b/packages/viewer-core/src/index.ts index 6f6dcda..84aacde 100644 --- a/packages/viewer-core/src/index.ts +++ b/packages/viewer-core/src/index.ts @@ -7,19 +7,99 @@ export interface DocumentMetadata { pageCount: number; } +export interface PageInfo { + pageNumber: number; + width: number; + height: number; +} + export interface RenderResult { width: number; height: number; imageDataUrl: string; } +export type LayerKind = 'base_pdf' | 'overlay_pdf' | 'markup_overlay' | 'selection_overlay'; + +export interface LayerVisibility { + visible: boolean; +} + +export interface LayerTransform { + offsetX: number; + offsetY: number; + rotationDeg?: number; +} + +export interface RenderLayer { + id: string; + kind: LayerKind; + documentKey?: string; + opacity: number; + visibility: LayerVisibility; + transform?: LayerTransform; +} + +export interface ViewportState { + scale: number; + scrollX: number; + scrollY: number; +} + +export interface RenderScene { + pageNumber: number; + viewport: ViewportState; + layers: RenderLayer[]; +} + +export interface PageRenderRequest { + pageNumber: number; + renderSpec: RenderSpec; + scene: RenderScene; +} + export interface ViewerRenderer { openDocument(source: Uint8Array): Promise; closeDocument(): Promise; getDocumentMetadata(): Promise; getPageCount(): Promise; + getPageInfo(pageNumber: number): Promise; renderPage(pageNumber: number, spec: RenderSpec): Promise; + renderLayer(layer: RenderLayer, pageNumber: number, spec: RenderSpec): Promise; getTextContent(pageNumber: number): Promise; screenToPage(x: number, y: number): { x: number; y: number }; pageToScreen(x: number, y: number): { x: number; y: number }; } + +export function createBaseRenderScene(pageNumber = 1, scale = 1): RenderScene { + return { + pageNumber, + viewport: { scale, scrollX: 0, scrollY: 0 }, + layers: [ + { + id: 'base', + kind: 'base_pdf', + opacity: 1, + visibility: { visible: true }, + transform: { offsetX: 0, offsetY: 0 } + } + ] + }; +} + +export function withOverlayPdfLayer(scene: RenderScene, overlayDocumentKey: string): RenderScene { + return { + ...scene, + layers: [ + ...scene.layers, + { + id: 'overlay', + kind: 'overlay_pdf', + documentKey: overlayDocumentKey, + opacity: 0.5, + visibility: { visible: true }, + transform: { offsetX: 0, offsetY: 0 } + } + ] + }; +} diff --git a/packages/viewer-pdfjs/src/index.ts b/packages/viewer-pdfjs/src/index.ts index 2a41174..3d9959c 100644 --- a/packages/viewer-pdfjs/src/index.ts +++ b/packages/viewer-pdfjs/src/index.ts @@ -1,6 +1,13 @@ import { GlobalWorkerOptions, getDocument, type PDFDocumentProxy } from 'pdfjs-dist'; import pdfWorker from 'pdfjs-dist/build/pdf.worker.min.mjs?url'; -import type { DocumentMetadata, RenderResult, RenderSpec, ViewerRenderer } from '@gitplant/viewer-core'; +import type { + DocumentMetadata, + PageInfo, + RenderLayer, + RenderResult, + RenderSpec, + ViewerRenderer +} from '@gitplant/viewer-core'; GlobalWorkerOptions.workerSrc = pdfWorker; @@ -28,7 +35,33 @@ export class PdfjsRendererAdapter implements ViewerRenderer { return this.document.numPages; } + async getPageInfo(pageNumber: number): Promise { + if (!this.document) throw new Error('No document opened'); + const page = await this.document.getPage(pageNumber); + const viewport = page.getViewport({ scale: 1 }); + return { pageNumber, width: viewport.width, height: viewport.height }; + } + async renderPage(pageNumber: number, spec: RenderSpec): Promise { + return this.renderBasePdfPage(pageNumber, spec); + } + + async renderLayer(layer: RenderLayer, pageNumber: number, spec: RenderSpec): Promise { + if (layer.kind === 'base_pdf' || layer.kind === 'overlay_pdf') { + return this.renderBasePdfPage(pageNumber, spec); + } + const page = await this.getPageInfo(pageNumber); + const canvas = document.createElement('canvas'); + canvas.width = page.width; + canvas.height = page.height; + return { + width: page.width, + height: page.height, + imageDataUrl: canvas.toDataURL('image/png') + }; + } + + private async renderBasePdfPage(pageNumber: number, spec: RenderSpec): Promise { if (!this.document) throw new Error('No document opened'); const page = await this.document.getPage(pageNumber); const viewport = page.getViewport({ scale: spec.scale });