From c1f40bf3ce0f606a1bfd3f58a43e8c27ace3ea07 Mon Sep 17 00:00:00 2001 From: Ada Date: Tue, 6 Jan 2026 19:30:21 -0500 Subject: [PATCH] =?UTF-8?q?SCMS=20[SCMS]=20Add=20admin=20endpoint=20to=20s?= =?UTF-8?q?ync=20Lab=20Notes=20from=20filesystem=20=F0=9F=94=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add POST /admin/notes/sync (auth-gated) - Read LABNOTES_DIR and ingest Markdown → HTML → DB upsert - Return deterministic JSON summary (scanned/upserted/skipped/errors) co-authored-by: Lyric co-authored-by: Carmel --- .env.example | 3 + package-lock.json | 72 ++++++++++++- package.json | 1 + src/routes/adminRoutes.ts | 13 +++ src/services/syncLabNotesFromFs.ts | 159 +++++++++++++++++++++++++++++ 5 files changed, 244 insertions(+), 4 deletions(-) create mode 100644 src/services/syncLabNotesFromFs.ts diff --git a/.env.example b/.env.example index 7d3104c..aaffe93 100644 --- a/.env.example +++ b/.env.example @@ -17,3 +17,6 @@ SESSION_SECRET=your_session_secret_here ALLOWED_GITHUB_USERNAME=your_username_here ADMIN_DEV_BYPASS=true +LABNOTES_DIR=/home/humanpatternlab/lab-api/content/labnotes + + diff --git a/package-lock.json b/package-lock.json index 5a48eaf..4fbe909 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "express": "^5.2.1", "express-openapi-validator": "^5.6.0", "express-session": "^1.18.2", + "gray-matter": "^4.0.3", "marked": "^17.0.1", "passport": "^0.7.0", "passport-github2": "^0.1.12" @@ -2381,7 +2382,6 @@ "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, "license": "MIT", "dependencies": { "sprintf-js": "~1.0.2" @@ -3456,7 +3456,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, "license": "BSD-2-Clause", "bin": { "esparse": "bin/esparse.js", @@ -3652,6 +3651,18 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "license": "MIT", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4038,6 +4049,21 @@ "dev": true, "license": "ISC" }, + "node_modules/gray-matter": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz", + "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==", + "license": "MIT", + "dependencies": { + "js-yaml": "^3.13.1", + "kind-of": "^6.0.2", + "section-matter": "^1.0.0", + "strip-bom-string": "^1.0.0" + }, + "engines": { + "node": ">=6.0" + } + }, "node_modules/handlebars": { "version": "4.7.8", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", @@ -4252,6 +4278,15 @@ "dev": true, "license": "MIT" }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -5013,7 +5048,6 @@ "version": "3.14.2", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", - "dev": true, "license": "MIT", "dependencies": { "argparse": "^1.0.7", @@ -5062,6 +5096,15 @@ "node": ">=6" } }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -6121,6 +6164,19 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/section-matter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", + "integrity": "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==", + "license": "MIT", + "dependencies": { + "extend-shallow": "^2.0.1", + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/semver": { "version": "7.7.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", @@ -6372,7 +6428,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/stack-utils": { @@ -6565,6 +6620,15 @@ "node": ">=8" } }, + "node_modules/strip-bom-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", + "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/strip-final-newline": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", diff --git a/package.json b/package.json index 9aedafb..7c11ef7 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "express": "^5.2.1", "express-openapi-validator": "^5.6.0", "express-session": "^1.18.2", + "gray-matter": "^4.0.3", "marked": "^17.0.1", "passport": "^0.7.0", "passport-github2": "^0.1.12" diff --git a/src/routes/adminRoutes.ts b/src/routes/adminRoutes.ts index 4c64a04..fc0dd97 100644 --- a/src/routes/adminRoutes.ts +++ b/src/routes/adminRoutes.ts @@ -3,6 +3,7 @@ import type { Request, Response } from "express"; import type Database from "better-sqlite3"; import { randomUUID } from "node:crypto"; import passport, { requireAdmin, isGithubOAuthEnabled } from "../auth.js"; +import { syncLabNotesFromFs } from "../services/syncLabNotesFromFs.js"; import { normalizeLocale, sha256Hex } from "../lib/helpers.js"; export function registerAdminRoutes(app: any, db: Database.Database) { @@ -398,6 +399,18 @@ export function registerAdminRoutes(app: any, db: Database.Database) { }); + // --------------------------------------------------------------------------- + // Admin: Syncs MD Files to DB (protected) + // --------------------------------------------------------------------------- + app.post("/admin/notes/sync", requireAdmin, (req: any, res: { json: (arg0: { rootDir: string; locales: string[]; scanned: number; upserted: number; skipped: number; errors: Array<{ file: string; error: string; }>; ok: boolean; }) => void; status: (arg0: number) => { (): any; new(): any; json: { (arg0: { ok: boolean; error: any; }): void; new(): any; }; }; }) => { + try { + const result = syncLabNotesFromFs(db); + res.json({ ok: true, ...result }); + } catch (e: any) { + res.status(500).json({ ok: false, error: e?.message ?? String(e) }); + } + }); + // --------------------------------------------------------------------------- // Auth helpers (always available) // --------------------------------------------------------------------------- diff --git a/src/services/syncLabNotesFromFs.ts b/src/services/syncLabNotesFromFs.ts new file mode 100644 index 0000000..3e1c17e --- /dev/null +++ b/src/services/syncLabNotesFromFs.ts @@ -0,0 +1,159 @@ +// src/services/syncLabNotesFromFs.ts +import fs from "node:fs"; +import path from "node:path"; +import matter from "gray-matter"; +import { marked } from "marked"; +import type Database from "better-sqlite3"; + +type SyncCounts = { + rootDir: string; + locales: string[]; + scanned: number; + upserted: number; + skipped: number; + errors: Array<{ file: string; error: string }>; +}; + +function listMarkdownFiles(dir: string): string[] { + if (!fs.existsSync(dir)) return []; + return fs + .readdirSync(dir) + .filter((f) => f.toLowerCase().endsWith(".md")) + .map((f) => path.join(dir, f)); +} + +function slugFromFilename(filePath: string): string { + return path.basename(filePath, path.extname(filePath)); +} + +export function syncLabNotesFromFs(db: Database.Database): SyncCounts { + const rootDir = String(process.env.LABNOTES_DIR || "").trim(); + if (!rootDir) { + throw new Error("LABNOTES_DIR is not set"); + } + if (!fs.existsSync(rootDir)) { + throw new Error(`LABNOTES_DIR not found: ${rootDir}`); + } + + const localeDirs = fs + .readdirSync(rootDir, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name); + + // Fallback: if root contains .md directly, treat as "en" + const rootMd = listMarkdownFiles(rootDir); + const locales = localeDirs.length ? localeDirs : ["en"]; + + const counts: SyncCounts = { + rootDir, + locales, + scanned: 0, + upserted: 0, + skipped: 0, + errors: [], + }; + + const upsert = db.prepare(` + INSERT INTO lab_notes ( + id, slug, title, excerpt, content_html, locale, + category, department_id, + shadow_density, coherence_score, safer_landing, read_time_minutes, + published_at + ) + VALUES ( + coalesce(?, lower(hex(randomblob(16)))), + ?, ?, ?, ?, ?, + ?, ?, + ?, ?, ?, ?, + ? + ) + ON CONFLICT(slug, locale) DO UPDATE SET + title=excluded.title, + excerpt=excluded.excerpt, + content_html=excluded.content_html, + category=excluded.category, + department_id=excluded.department_id, + shadow_density=excluded.shadow_density, + coherence_score=excluded.coherence_score, + safer_landing=excluded.safer_landing, + read_time_minutes=excluded.read_time_minutes, + published_at=excluded.published_at, + updated_at=CURRENT_TIMESTAMP + `); + + const selectExisting = db.prepare(` + SELECT content_html, title, excerpt, category, department_id + FROM lab_notes + WHERE slug = ? AND locale = ? + LIMIT 1 + `); + + const processFile = (filePath: string, locale: string) => { + counts.scanned += 1; + + try { + const raw = fs.readFileSync(filePath, "utf8"); + const parsed = matter(raw); + + const slug = String(parsed.data.slug || slugFromFilename(filePath)).trim(); + const title = String(parsed.data.title || slug).trim(); + + const excerpt = String(parsed.data.excerpt || "").trim(); + const category = parsed.data.category ? String(parsed.data.category) : null; + const departmentId = parsed.data.department_id ? String(parsed.data.department_id) : null; + + const shadowDensity = parsed.data.shadow_density ?? null; + const coherenceScore = parsed.data.coherence_score ?? null; + const saferLanding = parsed.data.safer_landing ?? null; + const readTimeMinutes = parsed.data.read_time_minutes ?? null; + const publishedAt = parsed.data.published_at ? String(parsed.data.published_at) : null; + + const contentHtml = marked.parse(String(parsed.content || "")); + + // Skip if nothing meaningfully changed (basic check) + const existing = selectExisting.get(slug, locale) as any; + if ( + existing && + existing.content_html === contentHtml && + existing.title === title && + existing.excerpt === excerpt && + (existing.category ?? null) === (category ?? null) && + (existing.department_id ?? null) === (departmentId ?? null) + ) { + counts.skipped += 1; + return; + } + + upsert.run( + null, // id (optional) + slug, + title, + excerpt || null, + contentHtml, + locale, + category, + departmentId, + shadowDensity, + coherenceScore, + saferLanding, + readTimeMinutes, + publishedAt + ); + + counts.upserted += 1; + } catch (e: any) { + counts.errors.push({ file: filePath, error: e?.message ?? String(e) }); + } + }; + + if (localeDirs.length) { + for (const loc of localeDirs) { + const files = listMarkdownFiles(path.join(rootDir, loc)); + for (const f of files) processFile(f, loc); + } + } else { + for (const f of rootMd) processFile(f, "en"); + } + + return counts; +}