Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions docs/guides/cron-schedule.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ fixed. You wake up and the brain is smarter than when you went to sleep.
| 3x/day (weekdays) | Meeting sync | Full ingestion + attendee propagation | [meeting-sync](../../recipes/meeting-sync.md) |
| Weekly | Calendar sync | Daily files + attendee enrichment | [calendar-to-brain](../../recipes/calendar-to-brain.md) |
| Daily AM | Morning briefing | Search calendar attendees, deal status, active threads | [briefing skill](../../skills/briefing/SKILL.md) |
| Weekly | Brain maintenance | `gbrain doctor`, embed stale, orphan detection | [maintain skill](../../skills/maintain/SKILL.md) |
| Nightly | Dream cycle | Entity sweep, enrich thin spots, fix citations | See below |
| Weekly | Brain maintenance | `gbrain doctor`, embed stale, orphan detection, relink DB graph from markdown links | [maintain skill](../../skills/maintain/SKILL.md) |
| Nightly | Dream cycle | Entity sweep, enrich thin spots, fix citations, add/fix cross-links | See below |

## Implementation: Setting Up Cron Jobs

Expand All @@ -42,7 +42,7 @@ fixed. You wake up and the brain is smarter than when you went to sleep.
0 10 * * 0 cd /path/to/calendar-sync && node calendar-sync.mjs --start $(date -v-7d +%Y-%m-%d) --end $(date +%Y-%m-%d)

# Brain health — weekly Mondays at 6 AM
0 6 * * 1 gbrain doctor --json >> /tmp/gbrain-health.log 2>&1 && gbrain embed --stale
0 6 * * 1 gbrain doctor --json >> /tmp/gbrain-health.log 2>&1 && gbrain embed --stale && gbrain relink --dir /path/to/brain

# Dream cycle — nightly at 2 AM
0 2 * * * /path/to/dream-cycle.sh
Expand Down Expand Up @@ -121,6 +121,7 @@ dream_cycle():

// Phase 4: Sync
gbrain sync --no-pull --no-embed
gbrain relink --dir /path/to/brain
gbrain embed --stale
```

Expand Down Expand Up @@ -182,9 +183,9 @@ echo "Dream cycle complete at $(date)"

1. **Quiet hours:** Set quiet hours to current hour. Run a notification cron.
Verify output went to `/tmp/cron-held/`, not to messaging.
2. **Dream cycle:** Run the dream cycle manually. Check that thin entity pages
got enriched and broken citations were fixed.
3. **Email collector cron:** Wait 30 minutes. Check `data/digests/` for new digest.
2. Dream cycle: Run the dream cycle manually. Check that thin entity pages
got enriched, broken citations were fixed, and the DB link graph was rebuilt.
3. Email collector cron: Wait 30 minutes. Check `data/digests/` for new digest.
4. **Morning briefing:** Check that held messages appear in the briefing.
5. **Health check:** Run `gbrain doctor --json`. All checks should pass.

Expand Down
8 changes: 7 additions & 1 deletion src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ for (const op of operations) {
}

// CLI-only commands that bypass the operation layer
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot']);
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'relink', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot']);

async function main() {
const args = process.argv.slice(2);
Expand Down Expand Up @@ -321,6 +321,11 @@ async function handleCliOnly(command: string, args: string[]) {
await runEmbed(engine, args);
break;
}
case 'relink': {
const { runRelink } = await import('./commands/relink.ts');
await runRelink(engine, args);
break;
}
case 'serve': {
const { runServe } = await import('./commands/serve.ts');
await runServe(engine);
Expand Down Expand Up @@ -468,6 +473,7 @@ TOOLS
extract <links|timeline|all> [dir] Extract links/timeline from markdown into DB
publish <page.md> [--password] Shareable HTML (strips private data, optional AES-256)
check-backlinks <check|fix> [dir] Find/fix missing back-links across brain
relink [--dir <brain-dir>] Rebuild DB link graph from markdown links
lint <dir|file> [--fix] Catch LLM artifacts, placeholder dates, bad frontmatter
report --type <name> --content ... Save timestamped report to brain/reports/

Expand Down
86 changes: 86 additions & 0 deletions src/commands/relink.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
import { existsSync, lstatSync, readdirSync } from 'fs';
import { join, relative } from 'path';
import type { BrainEngine } from '../core/engine.ts';
import { importFromFile } from '../core/import-file.ts';
import { isSyncable } from '../core/sync.ts';

function collectSyncableFiles(rootDir: string): { fullPath: string; relPath: string }[] {
const files: { fullPath: string; relPath: string }[] = [];

function walk(dir: string) {
for (const entry of readdirSync(dir)) {
if (entry.startsWith('.')) continue;
const fullPath = join(dir, entry);
const stat = lstatSync(fullPath);
if (stat.isSymbolicLink()) continue;
if (stat.isDirectory()) {
walk(fullPath);
continue;
}

const relPath = relative(rootDir, fullPath).replace(/\\/g, '/');
if (isSyncable(relPath)) {
files.push({ fullPath, relPath });
}
}
}

walk(rootDir);
files.sort((a, b) => a.relPath.localeCompare(b.relPath));
return files;
}

export async function runRelink(engine: BrainEngine, args: string[]) {
if (args.includes('--help') || args.includes('-h')) {
console.log(`gbrain relink — rebuild DB link graph from markdown links

USAGE
gbrain relink [--dir <brain-dir>] [--dry-run]

OPTIONS
--dir <path> Brain directory to scan (default: current directory)
--dry-run Report how many files would be relinked without writing

Relink re-imports every syncable markdown file under <dir> with embeddings
skipped, which triggers link reconciliation in import-file. Use after bulk
markdown edits, schema migrations, or when the DB link graph has drifted
from the markdown source of truth.`);
return;
}

const dirIdx = args.indexOf('--dir');
const rootDir = dirIdx >= 0 ? args[dirIdx + 1] : '.';
const dryRun = args.includes('--dry-run');

if (!existsSync(rootDir)) {
console.error(`Directory not found: ${rootDir}`);
process.exit(1);
}

const files = collectSyncableFiles(rootDir);
if (files.length === 0) {
console.log('No syncable markdown files found.');
return;
}

if (dryRun) {
console.log(`Would relink ${files.length} syncable markdown file(s) from ${rootDir}.`);
return;
}

let imported = 0;
let skipped = 0;
let errors = 0;

for (const file of files) {
const result = await importFromFile(engine, file.fullPath, file.relPath, { noEmbed: true });
if (result.status === 'imported') imported++;
else if (result.status === 'skipped') skipped++;
else errors++;
}

console.log(`Relinked ${files.length} file(s) from ${rootDir}.`);
console.log(` imported_or_updated: ${imported}`);
console.log(` unchanged_or_reconciled: ${skipped}`);
console.log(` errors: ${errors}`);
}
31 changes: 30 additions & 1 deletion src/core/import-file.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { readFileSync, statSync, lstatSync } from 'fs';
import { createHash } from 'crypto';
import type { BrainEngine } from './engine.ts';
import { parseMarkdown } from './markdown.ts';
import { extractInternalMarkdownLinks, parseMarkdown } from './markdown.ts';
import { chunkText } from './chunkers/recursive.ts';
import { embedBatch } from './embedding.ts';
import { slugifyPath } from './sync.ts';
Expand All @@ -16,6 +16,29 @@ export interface ImportResult {

const MAX_FILE_SIZE = 5_000_000; // 5MB

async function reconcilePageLinks(engine: BrainEngine, slug: string, desiredTargets: string[]): Promise<void> {
const desired = new Set(desiredTargets.filter(target => target && target !== slug));
const existingLinks = await engine.getLinks(slug);
const existing = new Set(existingLinks.map(link => link.to_slug));

for (const target of existing) {
if (!desired.has(target)) {
await engine.removeLink(slug, target);
}
}

for (const target of desired) {
if (!existing.has(target)) {
try {
await engine.addLink(slug, target);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.warn(`Skipping unresolved link ${slug} -> ${target}: ${message}`);
}
}
}
}

/**
* Import content from a string. Core pipeline:
* parse -> hash -> embed (external) -> transaction(version + putPage + tags + chunks)
Expand Down Expand Up @@ -48,6 +71,7 @@ export async function importFromContent(
}

const parsed = parseMarkdown(content, slug + '.md');
const desiredLinkTargets = extractInternalMarkdownLinks(content, slug).map(link => link.target_slug);

// Hash includes ALL fields for idempotency (not just compiled_truth + timeline)
const hash = createHash('sha256')
Expand All @@ -63,6 +87,9 @@ export async function importFromContent(

const existing = await engine.getPage(slug);
if (existing?.content_hash === hash) {
await engine.transaction(async (tx) => {
await reconcilePageLinks(tx, slug, desiredLinkTargets);
});
return { slug, status: 'skipped', chunks: 0 };
}

Expand Down Expand Up @@ -115,6 +142,8 @@ export async function importFromContent(
await tx.addTag(slug, tag);
}

await reconcilePageLinks(tx, slug, desiredLinkTargets);

if (chunks.length > 0) {
await tx.upsertChunks(slug, chunks);
} else {
Expand Down
62 changes: 61 additions & 1 deletion src/core/markdown.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { posix as pathPosix } from 'path';
import matter from 'gray-matter';
import type { PageType } from './types.ts';
import { slugifyPath } from './sync.ts';
Expand All @@ -12,6 +13,12 @@ export interface ParsedMarkdown {
tags: string[];
}

export interface InternalMarkdownLink {
text: string;
href: string;
target_slug: string;
}

/**
* Parse a markdown file with YAML frontmatter into its components.
*
Expand Down Expand Up @@ -84,14 +91,25 @@ export function splitBody(body: string): { compiled_truth: string; timeline: str
}

if (splitIndex === -1) {
return { compiled_truth: body, timeline: '' };
return splitBodyByHeading(body);
}

const compiled_truth = lines.slice(0, splitIndex).join('\n');
const timeline = lines.slice(splitIndex + 1).join('\n');
return { compiled_truth, timeline };
}

function splitBodyByHeading(body: string): { compiled_truth: string; timeline: string } {
const timelineMatch = body.match(/^##\s+Timeline\s*$/m);
if (!timelineMatch || timelineMatch.index === undefined) {
return { compiled_truth: body, timeline: '' };
}

const compiled_truth = body.slice(0, timelineMatch.index).trimEnd();
const timeline = body.slice(timelineMatch.index + timelineMatch[0].length).trimStart();
return { compiled_truth, timeline };
}

/**
* Serialize a page back to markdown format.
* Produces: frontmatter + compiled_truth + --- + timeline
Expand Down Expand Up @@ -122,6 +140,48 @@ export function serializeMarkdown(
return yamlContent + '\n\n' + body + '\n';
}

/**
* Extract internal markdown links and resolve them to page slugs.
* External URLs, anchors, mailto links, and non-markdown assets are ignored.
*/
export function extractInternalMarkdownLinks(content: string, sourceSlug: string): InternalMarkdownLink[] {
const linkPattern = /\[([^\]]+)\]\(([^)]+)\)/g;
const links: InternalMarkdownLink[] = [];
const seen = new Set<string>();

let match: RegExpExecArray | null;
while ((match = linkPattern.exec(content)) !== null) {
const text = match[1]?.trim() || '';
const href = match[2]?.trim() || '';
const targetSlug = resolveInternalMarkdownLink(sourceSlug, href);
if (!targetSlug) continue;

const dedupeKey = `${targetSlug}|${href}|${text}`;
if (seen.has(dedupeKey)) continue;
seen.add(dedupeKey);
links.push({ text, href, target_slug: targetSlug });
}

return links;
}

export function resolveInternalMarkdownLink(sourceSlug: string, href: string): string | null {
const trimmed = href.trim();
if (!trimmed) return null;
if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) return null;
if (trimmed.startsWith('mailto:') || trimmed.startsWith('#')) return null;

const withoutFragment = trimmed.split('#')[0] || '';
const withoutQuery = withoutFragment.split('?')[0] || '';
if (!withoutQuery.toLowerCase().endsWith('.md') && !withoutQuery.toLowerCase().endsWith('.mdx')) return null;

const sourcePath = `${sourceSlug}.md`;
const resolvedPath = pathPosix.normalize(pathPosix.join(pathPosix.dirname(sourcePath), withoutQuery));
if (resolvedPath.startsWith('..')) return null;

return slugifyPath(resolvedPath);
}

function inferType(filePath?: string): PageType {
if (!filePath) return 'concept';

Expand Down
Loading