diff --git a/.github/workflows/ci-host.yaml b/.github/workflows/ci-host.yaml index 3b6073f6cc..5a0fb629dc 100644 --- a/.github/workflows/ci-host.yaml +++ b/.github/workflows/ci-host.yaml @@ -21,7 +21,7 @@ on: permissions: checks: write - contents: read + contents: write id-token: write pull-requests: write @@ -219,6 +219,17 @@ jobs: name: host-test-report-${{ matrix.shardIndex }} path: junit/host-${{ matrix.shardIndex }}.xml retention-days: 30 + - name: Extract memory report + if: ${{ !cancelled() }} + run: node scripts/extract-memory-report.mjs /tmp/test-output.log /tmp/memory-report-${{ matrix.shardIndex }}.json + working-directory: packages/host + - name: Upload memory report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: ${{ !cancelled() }} + with: + name: memory-report-${{ matrix.shardIndex }} + path: /tmp/memory-report-${{ matrix.shardIndex }}.json + retention-days: 30 - name: Print realm server logs if: ${{ !cancelled() }} run: cat /tmp/server.log @@ -360,3 +371,29 @@ jobs: with: junit_files: host.xml check_name: Host Test Results + + - name: Download memory reports + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + if: ${{ !cancelled() }} + with: + path: all-memory-reports + pattern: memory-report-* + merge-multiple: true + + - name: Check memory baseline + if: ${{ !cancelled() }} + run: node packages/host/scripts/check-memory-baseline.mjs all-memory-reports packages/host/memory-baseline.json + + - name: Update memory baseline + if: ${{ !cancelled() && github.ref == 'refs/heads/main' && needs.host-test.result == 'success' }} + run: | + node packages/host/scripts/update-memory-baseline.mjs all-memory-reports packages/host/memory-baseline.json + if git diff --quiet packages/host/memory-baseline.json; then + echo "Baseline unchanged — nothing to commit." + else + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add packages/host/memory-baseline.json + git commit -m "Update host test memory baseline [skip ci]" + git push + fi diff --git a/packages/host/memory-baseline.json b/packages/host/memory-baseline.json new file mode 100644 index 0000000000..f1357e5661 --- /dev/null +++ b/packages/host/memory-baseline.json @@ -0,0 +1,520 @@ +{ + "version": 1, + "generated": "2026-04-15", + "threshold": { + "relative": 0.1, + "absolute_mb": 5 + }, + "modules": { + "Acceptance | AI Assistant tests": { + "delta_mb": 1540.4 + }, + "Acceptance | Code patches tests": { + "delta_mb": 402.9 + }, + "Acceptance | Commands tests": { + "delta_mb": 418.8 + }, + "Acceptance | Freestyle": { + "delta_mb": 8.6 + }, + "Acceptance | Query Fields | host respects server-populated results": { + "delta_mb": 19.7 + }, + "Acceptance | Spec preview": { + "delta_mb": 614.0 + }, + "Acceptance | avif image def": { + "delta_mb": 39.4 + }, + "Acceptance | basic tests": { + "delta_mb": 60.7 + }, + "Acceptance | code submode tests": { + "delta_mb": 752.4 + }, + "Acceptance | code submode | create-file tests": { + "delta_mb": 875.6 + }, + "Acceptance | code submode | editor tests": { + "delta_mb": 308.5 + }, + "Acceptance | code submode | file def live reload": { + "delta_mb": 42.0 + }, + "Acceptance | code submode | file def navigation": { + "delta_mb": 57.4 + }, + "Acceptance | code submode | file-tree tests": { + "delta_mb": 441.8 + }, + "Acceptance | code submode | head format preview": { + "delta_mb": 87.9 + }, + "Acceptance | code submode | inspector tests": { + "delta_mb": 1002.4 + }, + "Acceptance | code submode | recent files tests": { + "delta_mb": 119.2 + }, + "Acceptance | code submode | schema editor tests": { + "delta_mb": 389.8 + }, + "Acceptance | code-submode | card playground": { + "delta_mb": 859.5 + }, + "Acceptance | code-submode | field playground": { + "delta_mb": 411.6 + }, + "Acceptance | csv file def": { + "delta_mb": 56.4 + }, + "Acceptance | file chooser keyboard tests": { + "delta_mb": 1.8 + }, + "Acceptance | file chooser tests": { + "delta_mb": 82.7 + }, + "Acceptance | file chooser tests | upload size limit": { + "delta_mb": 0.1 + }, + "Acceptance | file def": { + "delta_mb": 68.1 + }, + "Acceptance | gif image def": { + "delta_mb": 38.7 + }, + "Acceptance | gts file def": { + "delta_mb": 37.8 + }, + "Acceptance | host mode tests": { + "delta_mb": 106.5 + }, + "Acceptance | host submode": { + "delta_mb": 187.9 + }, + "Acceptance | interact submode creation & permissions": { + "delta_mb": 266.5 + }, + "Acceptance | interact submode tests": { + "delta_mb": 469.1 + }, + "Acceptance | interact submode | create-file tests": { + "delta_mb": 170.0 + }, + "Acceptance | jpg image def": { + "delta_mb": 39.4 + }, + "Acceptance | json file def": { + "delta_mb": 37.3 + }, + "Acceptance | markdown BFM card references": { + "delta_mb": 95.8 + }, + "Acceptance | markdown file def": { + "delta_mb": 37.5 + }, + "Acceptance | operator mode tests": { + "delta_mb": 595.3 + }, + "Acceptance | png image def": { + "delta_mb": 39.6 + }, + "Acceptance | prerender | file-extract": { + "delta_mb": 37.3 + }, + "Acceptance | prerender | html": { + "delta_mb": 771.4 + }, + "Acceptance | prerender | meta": { + "delta_mb": 23.8 + }, + "Acceptance | prerender | module": { + "delta_mb": 26.0 + }, + "Acceptance | svg image def": { + "delta_mb": 41.6 + }, + "Acceptance | text file def": { + "delta_mb": 13.5 + }, + "Acceptance | theme-card-test": { + "delta_mb": 45.9 + }, + "Acceptance | ts file def": { + "delta_mb": 36.5 + }, + "Acceptance | webp image def": { + "delta_mb": 40.5 + }, + "Acceptance | workspace-chooser": { + "delta_mb": 13.7 + }, + "Acceptance | workspace-chooser-delete": { + "delta_mb": 32.8 + }, + "Acceptance | workspace-delete-multiple": { + "delta_mb": 11.8 + }, + "Integration | CardDef-FieldDef relationships test": { + "delta_mb": 50.4 + }, + "Integration | Command | create-specs": { + "delta_mb": 12.3 + }, + "Integration | Command | generate-example-cards (one-shot)": { + "delta_mb": -0.3 + }, + "Integration | Command | host command schema generation test": { + "delta_mb": 8.0 + }, + "Integration | Command | patch-fields": { + "delta_mb": 36.8 + }, + "Integration | Command | preview-format": { + "delta_mb": 25.2 + }, + "Integration | Command | show-card": { + "delta_mb": 0.0 + }, + "Integration | Command | update-room-skills": { + "delta_mb": 17.2 + }, + "Integration | Component | FormattedAiBotMessage": { + "delta_mb": 86.7 + }, + "Integration | Component | FormattedUserMessage": { + "delta_mb": 4.0 + }, + "Integration | Component | RoomMessage": { + "delta_mb": 6.8 + }, + "Integration | EmailField": { + "delta_mb": -5.4 + }, + "Integration | PhoneNumberField": { + "delta_mb": 4.7 + }, + "Integration | RichMarkdownField": { + "delta_mb": 178.1 + }, + "Integration | add-workspace": { + "delta_mb": 4.3 + }, + "Integration | ai-assistant-panel | binary upload dedupe": { + "delta_mb": 65.7 + }, + "Integration | ai-assistant-panel | codeblocks": { + "delta_mb": 108.6 + }, + "Integration | ai-assistant-panel | commands": { + "delta_mb": 487.2 + }, + "Integration | ai-assistant-panel | debug-message": { + "delta_mb": -8.4 + }, + "Integration | ai-assistant-panel | file-attachment": { + "delta_mb": 261.4 + }, + "Integration | ai-assistant-panel | general": { + "delta_mb": 405.0 + }, + "Integration | ai-assistant-panel | past sessions": { + "delta_mb": 72.5 + }, + "Integration | ai-assistant-panel | reasoning": { + "delta_mb": 45.2 + }, + "Integration | ai-assistant-panel | scrolling": { + "delta_mb": 80.6 + }, + "Integration | ai-assistant-panel | sending": { + "delta_mb": 100.8 + }, + "Integration | ai-assistant-panel | skills": { + "delta_mb": 235.3 + }, + "Integration | ask-ai": { + "delta_mb": 43.3 + }, + "Integration | card api (Usage of publicAPI actions)": { + "delta_mb": 17.6 + }, + "Integration | card-basics": { + "delta_mb": 54.9 + }, + "Integration | card-catalog": { + "delta_mb": 1.8 + }, + "Integration | card-copy": { + "delta_mb": 98.9 + }, + "Integration | card-delete": { + "delta_mb": 27.7 + }, + "Integration | codemirror-context": { + "delta_mb": 1.2 + }, + "Integration | commands | add-field-to-card-definition": { + "delta_mb": 1.1 + }, + "Integration | commands | ai-assistant": { + "delta_mb": 15.3 + }, + "Integration | commands | apply-markdown-edit": { + "delta_mb": 7.3 + }, + "Integration | commands | apply-search-replace-block": { + "delta_mb": 9.8 + }, + "Integration | commands | bot-registration": { + "delta_mb": 15.6 + }, + "Integration | commands | cancel-indexing-job": { + "delta_mb": 15.2 + }, + "Integration | commands | check-correctness": { + "delta_mb": 65.9 + }, + "Integration | commands | commands-calling": { + "delta_mb": -4.3 + }, + "Integration | commands | copy-and-edit": { + "delta_mb": 28.2 + }, + "Integration | commands | copy-card": { + "delta_mb": 42.0 + }, + "Integration | commands | copy-file-to-realm": { + "delta_mb": 6.1 + }, + "Integration | commands | copy-source": { + "delta_mb": 15.4 + }, + "Integration | commands | create-listing-pr-request": { + "delta_mb": -58.8 + }, + "Integration | commands | full-reindex-realm": { + "delta_mb": 7.4 + }, + "Integration | commands | get-user-system-card": { + "delta_mb": -22.8 + }, + "Integration | commands | invalidate-realm-urls": { + "delta_mb": -10.6 + }, + "Integration | commands | invite-user-to-room": { + "delta_mb": 12.7 + }, + "Integration | commands | open-create-listing-modal": { + "delta_mb": -8.8 + }, + "Integration | commands | open-workspace": { + "delta_mb": 14.0 + }, + "Integration | commands | patch-code": { + "delta_mb": 6.2 + }, + "Integration | commands | patch-instance": { + "delta_mb": 2.8 + }, + "Integration | commands | read-card-for-ai-assistant": { + "delta_mb": 13.9 + }, + "Integration | commands | read-file-for-ai-assistant": { + "delta_mb": 9.7 + }, + "Integration | commands | read-source": { + "delta_mb": 4.5 + }, + "Integration | commands | read-text-file": { + "delta_mb": -14.1 + }, + "Integration | commands | reindex-realm": { + "delta_mb": -12.9 + }, + "Integration | commands | search": { + "delta_mb": 17.4 + }, + "Integration | commands | search-and-choose": { + "delta_mb": 6.7 + }, + "Integration | commands | search-google-images": { + "delta_mb": -8.8 + }, + "Integration | commands | send-ai-assistant-message": { + "delta_mb": -1.2 + }, + "Integration | commands | send-bot-trigger-event": { + "delta_mb": 12.8 + }, + "Integration | commands | send-request-via-proxy": { + "delta_mb": 56.2 + }, + "Integration | commands | set-user-system-card": { + "delta_mb": 12.7 + }, + "Integration | commands | summarize-session": { + "delta_mb": 28.2 + }, + "Integration | commands | switch-submode": { + "delta_mb": 4.1 + }, + "Integration | commands | transform-cards": { + "delta_mb": 23.1 + }, + "Integration | commands | write-text-file": { + "delta_mb": -12.7 + }, + "Integration | components | create-listing-modal": { + "delta_mb": 293.5 + }, + "Integration | components | realm field": { + "delta_mb": -7.2 + }, + "Integration | computeds": { + "delta_mb": 13.9 + }, + "Integration | create app module via ai-assistant": { + "delta_mb": -0.0 + }, + "Integration | enumField": { + "delta_mb": 62.5 + }, + "Integration | field configuration": { + "delta_mb": 9.1 + }, + "Integration | loading": { + "delta_mb": 14.5 + }, + "Integration | markdown highlighting error scenarios": { + "delta_mb": 16.9 + }, + "Integration | message service subscription": { + "delta_mb": 11.8 + }, + "Integration | operator-mode | basics": { + "delta_mb": 356.1 + }, + "Integration | operator-mode | card catalog": { + "delta_mb": 208.6 + }, + "Integration | operator-mode | links": { + "delta_mb": 129.2 + }, + "Integration | operator-mode | ui": { + "delta_mb": 343.4 + }, + "Integration | overlay-menu-items": { + "delta_mb": 16.9 + }, + "Integration | prerendered-card-search": { + "delta_mb": 22.0 + }, + "Integration | preview": { + "delta_mb": 6.8 + }, + "Integration | realm": { + "delta_mb": 32.1 + }, + "Integration | realm indexing": { + "delta_mb": 3.7 + }, + "Integration | realm querying": { + "delta_mb": 39.6 + }, + "Integration | search data resource": { + "delta_mb": 18.1 + }, + "Integration | search resource": { + "delta_mb": 24.1 + }, + "Integration | serialization": { + "delta_mb": 29.2 + }, + "Integration | serializeFileDef": { + "delta_mb": 8.3 + }, + "Integration | text-input-validator": { + "delta_mb": 15.8 + }, + "Integration | text-suggestion | card-chooser-title": { + "delta_mb": 7.9 + }, + "Unit | CardDef menu items": { + "delta_mb": 4.1 + }, + "Unit | FileDef menu items": { + "delta_mb": 6.6 + }, + "Unit | Lib | command-definitions": { + "delta_mb": 0.0 + }, + "Unit | Lib | limited-set": { + "delta_mb": 0.0 + }, + "Unit | Utility | field-path-parser": { + "delta_mb": 0.1 + }, + "Unit | ai-function-generation-test": { + "delta_mb": 9.5 + }, + "Unit | auth-error-guard": { + "delta_mb": -0.0 + }, + "Unit | auth-service-worker": { + "delta_mb": 0.0 + }, + "Unit | bfm-card-references": { + "delta_mb": 0.1 + }, + "Unit | box": { + "delta_mb": 6.6 + }, + "Unit | cached-fetch": { + "delta_mb": -0.0 + }, + "Unit | code patching | parse search replace blocks": { + "delta_mb": 0.0 + }, + "Unit | convertToClassName": { + "delta_mb": 0.1 + }, + "Unit | diffing": { + "delta_mb": 0.0 + }, + "Unit | extract-css-variables": { + "delta_mb": 0.0 + }, + "Unit | fetcher": { + "delta_mb": -0.0 + }, + "Unit | file-def-manager canonicalize": { + "delta_mb": 0.0 + }, + "Unit | generate-css-variables": { + "delta_mb": 0.0 + }, + "Unit | identity-context garbage collection": { + "delta_mb": 8.1 + }, + "Unit | index-writer": { + "delta_mb": 68.5 + }, + "Unit | isolated-render": { + "delta_mb": 4.0 + }, + "Unit | last-modified-date": { + "delta_mb": 0.0 + }, + "Unit | query": { + "delta_mb": 3.2 + }, + "Unit | runtime-common | card directory names": { + "delta_mb": 0.0 + }, + "code-ref": { + "delta_mb": 9.2 + } + } +} diff --git a/packages/host/scripts/check-memory-baseline.mjs b/packages/host/scripts/check-memory-baseline.mjs new file mode 100644 index 0000000000..e2c348a6f0 --- /dev/null +++ b/packages/host/scripts/check-memory-baseline.mjs @@ -0,0 +1,158 @@ +#!/usr/bin/env node + +// Compares per-module memory deltas from a CI run against a committed baseline. +// Exits 0 on pass/warn, exits 1 on hard failure (>2x baseline or +50MB absolute). +// +// Usage: node check-memory-baseline.mjs +// +// contains per-shard memory-report.json files (merged into one dir). +// is the committed packages/host/memory-baseline.json. + +import { readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const [reportsDir, baselinePath] = process.argv.slice(2); + +if (!reportsDir || !baselinePath) { + console.error( + 'Usage: node check-memory-baseline.mjs ', + ); + process.exit(1); +} + +// Load and merge all shard reports +const current = {}; +for (const file of readdirSync(reportsDir)) { + if (!file.endsWith('.json')) continue; + const shard = JSON.parse(readFileSync(join(reportsDir, file), 'utf8')); + Object.assign(current, shard); +} + +// Load baseline +let baseline; +try { + baseline = JSON.parse(readFileSync(baselinePath, 'utf8')); +} catch (err) { + console.log(`No baseline found at ${baselinePath} — skipping check.`); + process.exit(0); +} + +const SOFT_RELATIVE = baseline.threshold?.relative ?? 0.10; +const SOFT_ABSOLUTE_MB = baseline.threshold?.absolute_mb ?? 5; +const HARD_RELATIVE = 1.0; // 2x = 100% increase +const HARD_ABSOLUTE_MB = 50; + +const warnings = []; +const failures = []; +const newModules = []; + +for (const [mod, data] of Object.entries(current)) { + if (mod === '__shard_warmup__') continue; + if (data.delta_mb == null) continue; + + const base = baseline.modules?.[mod]; + if (!base) { + newModules.push({ mod, delta: data.delta_mb }); + continue; + } + + const baseDelta = base.delta_mb; + if (baseDelta == null) continue; + + const diff = data.delta_mb - baseDelta; + const absDiff = Math.abs(diff); + // Only flag increases + if (diff <= 0) continue; + + const softThreshold = Math.max(SOFT_ABSOLUTE_MB, Math.abs(baseDelta) * SOFT_RELATIVE); + const hardThreshold = Math.max(HARD_ABSOLUTE_MB, Math.abs(baseDelta) * HARD_RELATIVE); + + if (absDiff >= hardThreshold) { + failures.push({ + mod, + baseline: baseDelta, + current: data.delta_mb, + diff, + pct: baseDelta !== 0 ? ((diff / Math.abs(baseDelta)) * 100).toFixed(0) : 'inf', + }); + } else if (absDiff >= softThreshold) { + warnings.push({ + mod, + baseline: baseDelta, + current: data.delta_mb, + diff, + pct: baseDelta !== 0 ? ((diff / Math.abs(baseDelta)) * 100).toFixed(0) : 'inf', + }); + } +} + +// Build summary +const lines = []; +lines.push('## Memory Baseline Check\n'); + +const totalModules = Object.keys(current).filter((m) => m !== '__shard_warmup__').length; +const baselineModules = Object.keys(baseline.modules || {}).length; +lines.push( + `Checked **${totalModules}** modules against baseline (${baselineModules} baselined).\n`, +); + +if (failures.length === 0 && warnings.length === 0) { + lines.push('All modules within threshold. No memory regressions detected.\n'); +} + +if (failures.length > 0) { + lines.push(`### Failures (>${HARD_RELATIVE * 100}% increase or +${HARD_ABSOLUTE_MB}MB)\n`); + lines.push('| Module | Baseline | Current | Change |'); + lines.push('|--------|----------|---------|--------|'); + for (const f of failures.sort((a, b) => b.diff - a.diff)) { + lines.push( + `| ${f.mod} | ${f.baseline.toFixed(1)} MB | ${f.current.toFixed(1)} MB | +${f.diff.toFixed(1)} MB (+${f.pct}%) |`, + ); + } + lines.push(''); +} + +if (warnings.length > 0) { + lines.push(`### Warnings (>${SOFT_RELATIVE * 100}% + ${SOFT_ABSOLUTE_MB}MB increase)\n`); + lines.push('| Module | Baseline | Current | Change |'); + lines.push('|--------|----------|---------|--------|'); + for (const w of warnings.sort((a, b) => b.diff - a.diff)) { + lines.push( + `| ${w.mod} | ${w.baseline.toFixed(1)} MB | ${w.current.toFixed(1)} MB | +${w.diff.toFixed(1)} MB (+${w.pct}%) |`, + ); + } + lines.push(''); +} + +if (newModules.length > 0) { + lines.push( + `
${newModules.length} new module(s) not in baseline\n`, + ); + for (const n of newModules.sort((a, b) => b.delta - a.delta)) { + lines.push(`- **${n.mod}**: ${n.delta.toFixed(1)} MB`); + } + lines.push('
\n'); +} + +const summary = lines.join('\n'); +console.log(summary); + +// Write to GITHUB_STEP_SUMMARY if available +if (process.env.GITHUB_STEP_SUMMARY) { + writeFileSync(process.env.GITHUB_STEP_SUMMARY, summary + '\n', { + flag: 'a', + }); +} + +if (failures.length > 0) { + console.error( + `\nFAILED: ${failures.length} module(s) exceeded hard memory threshold.`, + ); + process.exit(1); +} + +if (warnings.length > 0) { + console.log( + `\nWARN: ${warnings.length} module(s) exceeded soft memory threshold.`, + ); +} diff --git a/packages/host/scripts/extract-memory-report.mjs b/packages/host/scripts/extract-memory-report.mjs new file mode 100644 index 0000000000..2fe3e6c906 --- /dev/null +++ b/packages/host/scripts/extract-memory-report.mjs @@ -0,0 +1,60 @@ +#!/usr/bin/env node + +// Extracts MEMPROBE_FILE lines from a host test log and writes a JSON report. +// Handles both raw console.log format and testem's JSON-wrapped format. +// +// Usage: node extract-memory-report.mjs + +import { readFileSync, writeFileSync } from 'node:fs'; + +const [inputPath, outputPath] = process.argv.slice(2); + +if (!inputPath || !outputPath) { + console.error( + 'Usage: node extract-memory-report.mjs ', + ); + process.exit(1); +} + +const PROBE_RE = + /MEMPROBE_FILE module=("(?:[^"\\]|\\.)*"|\S+) tests=(\d+) used=([\d.]+)MB total=([\d.]+)MB delta=([\d.\-]+|na)MB/; +const JSON_ENVELOPE_RE = /\{"type":"log","text":"(.*?)"\}\s*$/; + +const log = readFileSync(inputPath, 'utf8'); +const report = {}; + +for (const rawLine of log.split('\n')) { + if (!rawLine.includes('MEMPROBE_FILE')) continue; + + let line = rawLine; + + // Unwrap testem JSON envelope if present + const envMatch = line.match(JSON_ENVELOPE_RE); + if (envMatch) { + try { + line = JSON.parse(`"${envMatch[1]}"`); + } catch { + // fall through to raw parse + } + } + + const m = line.match(PROBE_RE); + if (!m) continue; + + const mod = m[1].startsWith('"') ? JSON.parse(m[1]) : m[1]; + const deltaMb = m[5] === 'na' ? null : parseFloat(m[5]); + const usedMb = parseFloat(m[3]); + const totalMb = parseFloat(m[4]); + const tests = parseInt(m[2], 10); + + report[mod] = { delta_mb: deltaMb, used_mb: usedMb, total_mb: totalMb, tests }; +} + +writeFileSync(outputPath, JSON.stringify(report, null, 2) + '\n'); + +const count = Object.keys(report).length; +if (count === 0) { + console.log('extract-memory-report: no MEMPROBE_FILE lines found'); +} else { + console.log(`extract-memory-report: wrote ${count} modules to ${outputPath}`); +} diff --git a/packages/host/scripts/update-memory-baseline.mjs b/packages/host/scripts/update-memory-baseline.mjs new file mode 100644 index 0000000000..482afde762 --- /dev/null +++ b/packages/host/scripts/update-memory-baseline.mjs @@ -0,0 +1,48 @@ +#!/usr/bin/env node + +// Regenerates memory-baseline.json from per-shard memory reports. +// Used by CI on main merge (auto-update) or locally by developers. +// +// Usage: node update-memory-baseline.mjs + +import { readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const [reportsDir, baselinePath] = process.argv.slice(2); + +if (!reportsDir || !baselinePath) { + console.error( + 'Usage: node update-memory-baseline.mjs ', + ); + process.exit(1); +} + +// Merge all shard reports +const merged = {}; +for (const file of readdirSync(reportsDir)) { + if (!file.endsWith('.json')) continue; + const shard = JSON.parse(readFileSync(join(reportsDir, file), 'utf8')); + Object.assign(merged, shard); +} + +// Build baseline — exclude warmup since it varies by environment +const modules = {}; +for (const [mod, data] of Object.entries(merged).sort(([a], [b]) => + a.localeCompare(b), +)) { + if (mod === '__shard_warmup__') continue; + if (data.delta_mb == null) continue; + modules[mod] = { delta_mb: Math.round(data.delta_mb * 10) / 10 }; +} + +const baseline = { + version: 1, + generated: new Date().toISOString().slice(0, 10), + threshold: { relative: 0.1, absolute_mb: 5 }, + modules, +}; + +writeFileSync(baselinePath, JSON.stringify(baseline, null, 2) + '\n'); +console.log( + `update-memory-baseline: wrote ${Object.keys(modules).length} modules to ${baselinePath}`, +); diff --git a/packages/host/tests/helpers/setup-qunit.js b/packages/host/tests/helpers/setup-qunit.js index 90308008f1..e38e49b918 100644 --- a/packages/host/tests/helpers/setup-qunit.js +++ b/packages/host/tests/helpers/setup-qunit.js @@ -13,6 +13,57 @@ export function setupQUnit() { setup(QUnit.assert); QUnit.config.autostart = false; + // Per-module memory delta probe — log each test file's contribution to + // retained memory, independent of where it falls in shard order. We GC + // at module boundaries so the start/end snapshots compare like-for-like. + // QUnit module name is normally 1:1 with the test file. We only probe + // top-level modules (fullName.length === 1) so nested modules don't + // create overlapping start/end pairs. + let usedAtModuleStart = null; + let inTopLevelModule = false; + QUnit.on('suiteStart', (details) => { + let depth = Array.isArray(details.fullName) ? details.fullName.length : 1; + if (depth !== 1) return; + inTopLevelModule = true; + if (typeof globalThis.gc === 'function') { + globalThis.gc(); + globalThis.gc(); + } + try { + let pm = performance && performance.memory; + usedAtModuleStart = pm ? pm.usedJSHeapSize : null; + } catch (_) { + usedAtModuleStart = null; + } + }); + QUnit.on('suiteEnd', (details) => { + let depth = Array.isArray(details.fullName) ? details.fullName.length : 1; + if (depth !== 1 || !inTopLevelModule) return; + inTopLevelModule = false; + if (typeof globalThis.gc === 'function') { + globalThis.gc(); + globalThis.gc(); + } + try { + let pm = performance && performance.memory; + if (pm) { + let used = pm.usedJSHeapSize; + let usedMB = (used / 1048576).toFixed(1); + let totalMB = (pm.totalJSHeapSize / 1048576).toFixed(1); + let deltaMB = + usedAtModuleStart != null + ? ((used - usedAtModuleStart) / 1048576).toFixed(1) + : 'na'; + let tests = details.tests ? details.tests.length : 0; + console.log( + `MEMPROBE_FILE module=${JSON.stringify(details.name)} tests=${tests} used=${usedMB}MB total=${totalMB}MB delta=${deltaMB}MB`, + ); + } + } catch (_) { + /* ignore */ + } + }); + // After each test, force GC (via --expose-gc) so V8 can release // per-test allocations before the next test starts. Without this, V8's // opportunistic GC can't keep up and the heap drifts toward the 4GB diff --git a/packages/host/tests/helpers/shard-warmup.ts b/packages/host/tests/helpers/shard-warmup.ts new file mode 100644 index 0000000000..86f2dae78b --- /dev/null +++ b/packages/host/tests/helpers/shard-warmup.ts @@ -0,0 +1,67 @@ +// Synthetic warmup module — registered first on every shard so that the +// per-shard "boot cost" (Ember app boot, base-realm imports, mock matrix, +// initial test realm setup) lands on this module rather than on whichever +// real test file ember-exam happens to schedule first. The MEMPROBE_FILE +// line for this module then becomes the "shard boot cost" baseline, and +// real modules report a clean per-file delta independent of position. +// +// Imported via test-helper.js; the static import causes the module() call +// below to register with QUnit before ember-exam loads the partition's +// test files. + +import { visit } from '@ember/test-helpers'; + +import { getService } from '@universal-ember/test-support'; +import { module, test } from 'qunit'; + +import { baseRealm } from '@cardstack/runtime-common'; + +import { + SYSTEM_CARD_FIXTURE_CONTENTS, + setupAcceptanceTestRealm, + setupAuthEndpoints, + setupLocalIndexing, + setupUserSubscription, + testRealmURL, +} from './index'; +import { setupMockMatrix } from './mock-matrix'; +import { setupApplicationTest } from './setup'; + +module('__shard_warmup__', function (hooks) { + setupApplicationTest(hooks); + setupLocalIndexing(hooks); + + let mockMatrixUtils = setupMockMatrix(hooks, { + loggedInAs: '@testuser:localhost', + activeRealms: [testRealmURL], + }); + + let { createAndJoinRoom } = mockMatrixUtils; + + hooks.beforeEach(async function () { + createAndJoinRoom({ + sender: '@testuser:localhost', + name: 'room-warmup', + }); + setupUserSubscription(); + setupAuthEndpoints(); + + let loaderService = getService('loader-service'); + let loader = loaderService.loader; + // Prime the loader with the most commonly imported base-realm modules + // so subsequent real tests don't pay the import cost. + await loader.import(`${baseRealm.url}card-api`); + await loader.import(`${baseRealm.url}string`); + await loader.import(`${baseRealm.url}spec`); + + await setupAcceptanceTestRealm({ + mockMatrixUtils, + contents: { ...SYSTEM_CARD_FIXTURE_CONTENTS }, + }); + }); + + test('warm boot the test environment', async function (assert) { + await visit('/'); + assert.ok(true, 'shard warmup completed'); + }); +}); diff --git a/packages/host/tests/test-helper.js b/packages/host/tests/test-helper.js index 84b415ad9d..c08ac052bb 100644 --- a/packages/host/tests/test-helper.js +++ b/packages/host/tests/test-helper.js @@ -7,6 +7,10 @@ import start from 'ember-exam/test-support/start'; // eslint-disable-next-line ember/no-test-import-export import { loadRealmTests } from './live-test'; import { setupQUnit } from './helpers/setup-qunit'; +// Side-effect import: registers a `__shard_warmup__` QUnit module so the +// per-shard boot cost is absorbed by it rather than by whichever real +// test module happens to be scheduled first on the partition. +import './helpers/shard-warmup'; const application = Application.create({ ...config.APP,