docs: external perf notes + verification

.github/workflows/perf-baseline-update.yml

@@ -0,0 +1,91 @@
+name: Update Performance Baseline
+
+on:
+  push:
+    branches: [master]
+    paths:
+      - 'server/src/**'
+      - 'client/src/**'
+      - 'protocol/**'
+
+  workflow_dispatch:
+
+jobs:
+  update-baseline:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: oven-sh/setup-bun@v1
+        with:
+          bun-version: latest
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Map local.hytopiahosting.com to localhost
+        run: echo "127.0.0.1 local.hytopiahosting.com" | sudo tee -a /etc/hosts
+
+      - name: Install server dependencies
+        run: cd server && npm ci
+
+      - name: Build SDK
+        run: cd server && npm run build
+
+      - name: Install perf-tools
+        run: cd packages/perf-tools && PUPPETEER_SKIP_DOWNLOAD=true npm ci && npm run build
+
+      - name: Run benchmarks (3 rounds, averaged)
+        run: |
+          mkdir -p .perf-baseline
+          for preset in idle stress; do
+            echo "=== Running ${preset} benchmark (3 rounds) ==="
+            for i in 1 2 3; do
+              cd packages/perf-tools
+              npx hytopia-bench run --preset ${preset} --output "../../.perf-baseline/${preset}-run${i}.json" --verbose || true
+              cd ../..
+            done
+          done
+
+      - name: Average baselines
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const dir = '.perf-baseline';
+            const presets = ['idle', 'stress'];
+            for (const preset of presets) {
+              const runs = [];
+              for (let i = 1; i <= 3; i++) {
+                const path = `${dir}/${preset}-run${i}.json`;
+                if (fs.existsSync(path)) {
+                  const data = JSON.parse(fs.readFileSync(path, 'utf-8'));
+                  if (data.baseline) runs.push(data.baseline);
+                }
+              }
+              if (runs.length === 0) continue;
+              const avg = {};
+              const keys = ['avgTickMs', 'maxTickMs', 'p95TickMs', 'p99TickMs', 'ticksOverBudgetPct', 'avgMemoryMb'];
+              for (const key of keys) {
+                const values = runs.map(r => r[key]).filter(v => v !== undefined);
+                avg[key] = values.length > 0 ? values.reduce((a, b) => a + b, 0) / values.length : 0;
+              }
+              avg.operations = runs[0].operations || {};
+              fs.writeFileSync(`${dir}/${preset}.json`, JSON.stringify(avg, null, 2));
+              console.log(`${preset} baseline averaged from ${runs.length} runs`);
+            }
+
+      - name: Cache baseline
+        uses: actions/cache/save@v4
+        with:
+          path: .perf-baseline/
+          key: perf-baseline-master-${{ github.sha }}
+
+      - name: Also save as latest
+        uses: actions/cache/save@v4
+        with:
+          path: .perf-baseline/
+          key: perf-baseline-master

.github/workflows/perf-gate.yml

@@ -0,0 +1,122 @@
+name: Performance Gate
+
+on:
+  pull_request:
+    branches: [master]
+    paths:
+      - 'server/src/**'
+      - 'client/src/**'
+      - 'protocol/**'
+
+jobs:
+  perf-benchmark:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: oven-sh/setup-bun@v1
+        with:
+          bun-version: latest
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Map local.hytopiahosting.com to localhost
+        run: echo "127.0.0.1 local.hytopiahosting.com" | sudo tee -a /etc/hosts
+
+      - name: Install server dependencies
+        run: cd server && npm ci
+
+      - name: Build SDK
+        run: cd server && npm run build
+
+      - name: Install perf-tools
+        run: cd packages/perf-tools && PUPPETEER_SKIP_DOWNLOAD=true npm ci && npm run build
+
+      - name: Restore baseline cache
+        id: cache-baseline
+        uses: actions/cache/restore@v4
+        with:
+          path: .perf-baseline/
+          key: perf-baseline-${{ github.base_ref }}
+
+      - name: Run idle benchmark
+        run: |
+          cd packages/perf-tools
+          npx hytopia-bench run --preset idle --output ../../.perf-results/idle.json --verbose
+        continue-on-error: true
+
+      - name: Run stress benchmark
+        run: |
+          cd packages/perf-tools
+          npx hytopia-bench run --preset stress --output ../../.perf-results/stress.json --verbose
+        continue-on-error: true
+
+      - name: Compare with baseline
+        if: steps.cache-baseline.outputs.cache-hit == 'true'
+        run: |
+          cd packages/perf-tools
+          RESULTS=""
+          for preset in idle stress; do
+            if [ -f "../../.perf-baseline/${preset}.json" ] && [ -f "../../.perf-results/${preset}.json" ]; then
+              echo "=== Comparing ${preset} ==="
+              npx hytopia-bench compare \
+                "../../.perf-baseline/${preset}.json" \
+                "../../.perf-results/${preset}.json" \
+                --warn 5 --fail 10 || true
+            fi
+          done
+
+      - name: Post results as PR comment
+        if: always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const results = [];
+            const dir = '.perf-results';
+            if (fs.existsSync(dir)) {
+              for (const file of fs.readdirSync(dir)) {
+                if (file.endsWith('.json')) {
+                  const data = JSON.parse(fs.readFileSync(`${dir}/${file}`, 'utf-8'));
+                  results.push(data);
+                }
+              }
+            }
+            if (results.length === 0) {
+              console.log('No benchmark results to post');
+              return;
+            }
+            let body = '## Performance Benchmark Results\n\n';
+            for (const r of results) {
+              body += `### ${r.scenario}\n`;
+              body += `| Metric | Value |\n|--------|-------|\n`;
+              if (r.baseline) {
+                const b = r.baseline;
+                body += `| Avg Tick | ${b.avgTickMs?.toFixed(2) ?? 'N/A'}ms |\n`;
+                body += `| P95 Tick | ${b.p95TickMs?.toFixed(2) ?? 'N/A'}ms |\n`;
+                body += `| P99 Tick | ${b.p99TickMs?.toFixed(2) ?? 'N/A'}ms |\n`;
+                body += `| Max Tick | ${b.maxTickMs?.toFixed(2) ?? 'N/A'}ms |\n`;
+                body += `| Over Budget | ${b.ticksOverBudgetPct?.toFixed(1) ?? 'N/A'}% |\n`;
+                body += `| Avg Memory | ${b.avgMemoryMb?.toFixed(1) ?? 'N/A'}MB |\n`;
+              }
+              body += '\n';
+            }
+            body += '\n---\n*Generated by @hytopia/perf-tools*\n';
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body,
+            });
+
+      - name: Upload results artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: perf-results
+          path: .perf-results/
+          retention-days: 30

.gitignore

@@ -1 +1,5 @@
-node_modules/
+node_modules/
+server/src/playground.mjs
+server/src/perf-harness.mjs
+packages/*/dist/
+packages/perf-tools/perf-results/*/

CLAUDE.md

@@ -20,7 +20,7 @@ HYTOPIA is a multiplayer voxel game engine monorepo. The **server** (TypeScript/
 - **Physics**: Rapier3D (`@dimforge/rapier3d-simd-compat`) at 60 Hz, default gravity `y = -32`
 - **Networking**: WebTransport (QUIC) preferred, WebSocket fallback. Packets serialized with msgpackr, large payloads gzip-compressed
 - **Protocol**: `protocol/` defines all packet schemas (AJV-validated). Published as `@hytopia.com/server-protocol`
-- **Rendering**: Three.js `WebGLRenderer` + `MeshBasicMaterial` (no dynamic lights). Post-processing: SMAA, bloom, outline. Chunk meshes built in Web Worker via greedy meshing + AO
+- **Rendering**: Three.js `WebGLRenderer` + `MeshBasicMaterial` (no dynamic lights). Post-processing: SMAA, bloom, outline. Chunk meshes built in a Web Worker with face culling + AO (no greedy quad merging)
 - **Persistence**: `@hytopia.com/save-states` for player/global KV data
 - **Singleton pattern**: Most server systems use `ClassName.instance`; client systems owned by `Game` singleton
 

CODEBASE_DOCUMENTATION.md

@@ -16,6 +16,7 @@ PROTOCOL: protocol/ - Packet schemas + definitions (@hytopia.com/server-protocol
 SDK:      sdk/ - Git submodule → hytopiagg/sdk (build output lands here)
 EXAMPLES: sdk-examples/ - Reference games built with the SDK
 ASSETS:   assets/release/ - Default game assets (audio, blocks, maps, models, particles, skyboxes, ui)
+PERF:     packages/perf-tools/ - Benchmark CLI + trace analysis (`hytopia-bench`), headless client metrics, synthetic + real-game presets (`zoo-game-full` single-client benchmark, `zoo-game-observe` 5-client joinable Zoo run), helper scripts for linking/running external games (including linked SDK runtime deps, target-ref dependency prep via `ensure-node-modules.sh`, `apply-instrumentation-overlay.sh` for temporary legacy-ref PerfBridge/PerfHarness patching, `overlays/legacy-server/` for older PerfBaseline refs, `overlays/minimal-server/` for telemetry-minimal refs that need injected monitor/network hooks, repeatable HyFire2/Zoo Game workflows, and `run-owned-stack-suite.sh` for one-command multi-game runs against a chosen engine ref/PR with `origin`/`upstream` fetch fallback). The perf CLI now supports repeated-run median workflows via `BenchmarkSeriesAggregator.ts`, `hytopia-bench aggregate`, `hytopia-bench compare-series`, and `run-owned-stack-suite.sh --repeat <n>`, so noisy real-game/client scenarios can be judged on stable medians instead of one-off runs. Older engine refs now prefer overlayed client/server perf hooks, normalize report outputs to stable paths, then fall back to validated legacy `/__perf` normalization plus compare-time metric skipping instead of misleading zero baselines.
 CONFIG:   package.json - Monorepo root (npm workspaces)
           server/package.json - Server deps + build scripts
           server/tsconfig.json - Strict TS, path alias @/* → ./src/*
@@ -140,9 +141,18 @@ shared/types/math/Vector3Like.ts - Vector3 interface
 errors/ErrorHandler.ts - Fatal error handling + crash protection
 events/EventRouter.ts - Typed event emitter (eventemitter3)
 events/Events.ts - Event payload type definitions
+bots/BotManager.ts, bots/BotPlayer.ts - Server-side bot players for perf/stress tests
+metrics/Monitor.ts - @Monitor decorators + helper wrappers
+metrics/CpuProfiler.ts - V8 CPU profile + heap snapshot capture (debug tooling)
+metrics/PerformanceMonitor.ts - Tick profiler + operation percentiles + spikes
+metrics/NetworkMetrics.ts - Byte/packet/serialization counters
 metrics/Telemetry.ts - Span-based performance profiling
 models/ModelRegistry.ts - GLTF model preloading + bounding box extraction
 persistence/PersistenceManager.ts - Player/global KV storage via @hytopia.com/save-states
+perf/PerfHarness.ts - Env-gated /__perf endpoints for perf-tools
+perf/PerfBlockChurner.ts - Tick-driven block churn stressor (perf-tools)
+perf/PerfWorldGenerator.ts - Synthetic block-world generator for perf-tools scenarios
+perf/perf-harness.ts - Benchmark server entry (build:perf-harness → src/perf-harness.mjs)
 server/src/assets/AssetsLibrary.ts - Asset path resolution
 ```
 
@@ -423,4 +433,4 @@ zombies-fps/ - Zombie FPS
 - **Dual transport** — WebTransport (QUIC) preferred, WebSocket fallback. Reliable stream + unreliable datagrams
 - **msgpackr serialization** — All packets serialized with msgpackr, large payloads gzip-compressed
 - **60 Hz physics / 30 Hz network** — Server physics ticks at 60 Hz, network sync flushes every 2 ticks
-- **Web Worker meshing** — Client offloads greedy meshing + AO to a dedicated Web Worker
+- **Web Worker meshing** — Client offloads face-culling meshing + AO to a dedicated Web Worker (no greedy quad merging)

README.md

@@ -48,7 +48,7 @@ The `Game` singleton owns all subsystem managers. Key systems:
 |---|---|
 | `NetworkManager` | WebTransport (HTTP/3) with WebSocket fallback. Deserializes msgpack packets and dispatches typed events |
 | `Renderer` | Three.js `WebGLRenderer` + `EffectComposer`. Post-processing: SMAA, selective bloom, outline pass, `CSS2DRenderer` for in-world UI |
-| `ChunkMeshManager` + `ChunkWorkerClient` | Voxel mesh generation via greedy meshing with ambient occlusion, offloaded to a Web Worker |
+| `ChunkMeshManager` + `ChunkWorkerClient` | Voxel mesh generation with face culling + ambient occlusion, offloaded to a Web Worker (no greedy quad merging) |
 | `EntityManager` | Entity lifecycle and GLTF model rendering |
 | `InputManager` + `MobileManager` | Keyboard/mouse/gamepad input and touch/joystick for mobile |
 | `UIManager` | HTML/CSS overlay UI system for game developer UIs and in-world `SceneUI` elements |