diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c597bc2..dd07589 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,8 +41,17 @@ jobs: cache: npm registry-url: 'https://registry.npmjs.org' + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: npm ci --prefer-offline --no-audit + - name: Install Python dependencies + run: | + cd tywrap_ir + pip install -e . + - run: npm run build - name: Run tests diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 5419ec0..8b1b814 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -328,34 +328,41 @@ Hooks are optional; implement only what your plugin needs. ## Environment Variables -Override configuration with environment variables: +Most tywrap behavior is configured in `tywrap.config.*` or when you construct a +runtime bridge in application code. The supported `TYWRAP_*` environment +variables are mostly codec guardrails, logging, and repo test knobs: ```bash -# Runtime configuration -export TYWRAP_PYTHON_PATH="/usr/local/bin/python3.11" -export TYWRAP_VIRTUAL_ENV="./venv" export TYWRAP_CODEC_FALLBACK="json" -export TYWRAP_CODEC_MAX_BYTES="10485760" # Max response payload size (bytes) +export TYWRAP_CODEC_MAX_BYTES="10485760" # Max response payload size (bytes) export TYWRAP_REQUEST_MAX_BYTES="1048576" # Max request payload size (bytes) export TYWRAP_TORCH_ALLOW_COPY="1" +export TYWRAP_LOG_LEVEL="INFO" +export TYWRAP_LOG_JSON="1" +``` + +Repo tests and benchmarks also use additional `TYWRAP_*` variables such as +`TYWRAP_PERF_BUDGETS`. + +Python executable and virtual environment selection are not configured through +environment variables today. Set them in `tywrap.config.*` or on the bridge: -# Note: NodeBridge uses TYWRAP_CODEC_MAX_BYTES as the default maxLineLength when set. - -# Performance tuning -export TYWRAP_CACHE_DIR="./.tywrap/cache" -export TYWRAP_MEMORY_LIMIT="1024" -export TYWRAP_PERF_BUDGETS="1" -export TYWRAP_PERF_TIME_BUDGET_MS="2000" -export TYWRAP_PERF_MEMORY_BUDGET_MB="64" -export TYWRAP_CODEC_PERF_ITERATIONS="200" -export TYWRAP_CODEC_PERF_TIME_BUDGET_MS="500" -export TYWRAP_CODEC_PERF_MEMORY_BUDGET_MB="32" - -# Development -export TYWRAP_VERBOSE="true" -export TYWRAP_HOT_RELOAD="true" +```ts +import { defineConfig } from 'tywrap'; + +export default defineConfig({ + runtime: { + node: { + pythonPath: '/usr/local/bin/python3', + virtualEnv: './venv', + timeout: 30000, + }, + }, +}); ``` +See [Environment Variables](/reference/env-vars) for the full implemented list. + ## Advanced Configuration Patterns ### Multi-Environment Setup diff --git a/docs/guide/runtimes/node.md b/docs/guide/runtimes/node.md index 01bbcad..31a5d3a 100644 --- a/docs/guide/runtimes/node.md +++ b/docs/guide/runtimes/node.md @@ -290,19 +290,30 @@ try { ### Debugging Configuration -```json -{ - "runtime": { - "node": { - "timeoutMs": 0, // Disable timeout for debugging - "env": { - "PYTHONUNBUFFERED": "1", // Immediate stdout/stderr - "TYWRAP_DEBUG": "1" // Enable debug logging - } - } +Use the CLI's debug flag when you are troubleshooting wrapper generation: + +```bash +npx tywrap generate --debug +``` + +Use runtime log env vars for subprocess diagnostics: + +```bash +export TYWRAP_LOG_LEVEL=DEBUG +export TYWRAP_LOG_JSON=1 +``` + +If you need to disable timeouts or pass extra Python env vars while debugging, +do it on the bridge instance: + +```typescript +const bridge = new NodeBridge({ + pythonPath: 'python3', + timeoutMs: 0, + env: { + PYTHONUNBUFFERED: '1', }, - "debug": true -} +}); ``` ### Common Error Scenarios @@ -359,17 +370,16 @@ const [sin1, sin2, sin3] = await Promise.all([ ### Memory Management -```json -{ - "runtime": { - "node": { - "env": { - "PYTHONMALLOC": "malloc", // Use system malloc - "OMP_NUM_THREADS": "4" // Limit OpenMP threads - } - } - } -} +Pass Python-specific tuning through `env` on the bridge: + +```typescript +const bridge = new NodeBridge({ + pythonPath: 'python3', + env: { + PYTHONMALLOC: 'malloc', + OMP_NUM_THREADS: '4', + }, +}); ``` ## Production Deployment @@ -437,8 +447,8 @@ services: app: build: . environment: - - TYWRAP_PYTHON_PATH=/usr/bin/python3 - TYWRAP_CODEC_FALLBACK=json # For smaller containers + - TYWRAP_LOG_LEVEL=INFO ports: - '3000:3000' ``` @@ -448,32 +458,36 @@ services: ```bash # Production environment export NODE_ENV=production -export TYWRAP_PYTHON_PATH="/usr/local/bin/python3" -export TYWRAP_CACHE_DIR="/tmp/tywrap-cache" -export TYWRAP_MEMORY_LIMIT="2048" +export TYWRAP_CODEC_MAX_BYTES=10485760 +export TYWRAP_REQUEST_MAX_BYTES=1048576 +export TYWRAP_LOG_LEVEL=INFO # Security export PYTHONDONTWRITEBYTECODE=1 export PYTHONUNBUFFERED=1 ``` +Set the Python executable in config or when you construct the bridge: + +```typescript +const bridge = new NodeBridge({ + pythonPath: '/usr/local/bin/python3', +}); +``` + ## Security Considerations ### Subprocess Security -```json -{ - "runtime": { - "node": { - "cwd": "/safe/directory", // Restrict working directory - "env": { - "PATH": "/usr/bin:/bin", // Limit PATH - "PYTHONPATH": "/safe/python/libs" - }, - "timeoutMs": 10000 // Prevent hanging processes - } - } -} +```typescript +const bridge = new NodeBridge({ + cwd: '/safe/directory', + timeoutMs: 10000, + env: { + PATH: '/usr/bin:/bin', + PYTHONPATH: '/safe/python/libs', + }, +}); ``` ### Input Validation @@ -536,25 +550,25 @@ chmod +x /usr/local/bin/python3 **"Process timeout"**: -```json -{ - "runtime": { - "node": { - "timeoutMs": 60000, // Increase timeout - "env": { - "OMP_NUM_THREADS": "1" // Reduce parallelism - } - } - } -} +```typescript +const bridge = new NodeBridge({ + pythonPath: 'python3', + timeoutMs: 60000, + env: { + OMP_NUM_THREADS: '1', + }, +}); ``` ### Debug Mode ```bash -# Enable debug logging -export TYWRAP_DEBUG=1 -export TYWRAP_VERBOSE=1 +# Wrapper-generation diagnostics +npx tywrap generate --debug + +# Runtime bridge diagnostics +export TYWRAP_LOG_LEVEL=DEBUG +export TYWRAP_LOG_JSON=1 # Run with debug output node --trace-warnings your-app.js @@ -591,14 +605,11 @@ if __name__ == '__main__': pass ``` -```json -{ - "runtime": { - "node": { - "scriptPath": "./custom_bridge.py" - } - } -} +```typescript +const bridge = new NodeBridge({ + pythonPath: 'python3', + scriptPath: './custom_bridge.py', +}); ``` ### Process Pooling diff --git a/docs/index.md b/docs/index.md index aa70c32..4197b60 100644 --- a/docs/index.md +++ b/docs/index.md @@ -46,6 +46,6 @@ setRuntimeBridge(new NodeBridge({ pythonPath: 'python3' })); const result = await math.sqrt(16); // 4 — fully typed ``` -> ⚠️ **Experimental** — APIs may change before v1.0.0. See [CHANGELOG](https://github.com/bbopen/tywrap/blob/main/CHANGELOG.md) for breaking changes. +> ⚠️ **Experimental** — APIs may change before v1.0.0. See [Releases](https://github.com/bbopen/tywrap/releases) for breaking changes. > If tywrap saves you time, a ⭐ on [GitHub](https://github.com/bbopen/tywrap) helps others find it. diff --git a/docs/public/llms-full.txt b/docs/public/llms-full.txt index b89b210..1af4824 100644 --- a/docs/public/llms-full.txt +++ b/docs/public/llms-full.txt @@ -52,7 +52,7 @@ setRuntimeBridge(new NodeBridge({ pythonPath: 'python3' })); const result = await math.sqrt(16); // 4 — fully typed ``` -> ⚠️ **Experimental** — APIs may change before v1.0.0. See [CHANGELOG](https://github.com/bbopen/tywrap/blob/main/CHANGELOG.md) for breaking changes. +> ⚠️ **Experimental** — APIs may change before v1.0.0. See [Releases](https://github.com/bbopen/tywrap/releases) for breaking changes. > If tywrap saves you time, a ⭐ on [GitHub](https://github.com/bbopen/tywrap) helps others find it. @@ -769,34 +769,41 @@ Hooks are optional; implement only what your plugin needs. ## Environment Variables -Override configuration with environment variables: +Most tywrap behavior is configured in `tywrap.config.*` or when you construct a +runtime bridge in application code. The supported `TYWRAP_*` environment +variables are mostly codec guardrails, logging, and repo test knobs: ```bash -# Runtime configuration -export TYWRAP_PYTHON_PATH="/usr/local/bin/python3.11" -export TYWRAP_VIRTUAL_ENV="./venv" export TYWRAP_CODEC_FALLBACK="json" -export TYWRAP_CODEC_MAX_BYTES="10485760" # Max response payload size (bytes) +export TYWRAP_CODEC_MAX_BYTES="10485760" # Max response payload size (bytes) export TYWRAP_REQUEST_MAX_BYTES="1048576" # Max request payload size (bytes) export TYWRAP_TORCH_ALLOW_COPY="1" +export TYWRAP_LOG_LEVEL="INFO" +export TYWRAP_LOG_JSON="1" +``` -# Note: NodeBridge uses TYWRAP_CODEC_MAX_BYTES as the default maxLineLength when set. +Repo tests and benchmarks also use additional `TYWRAP_*` variables such as +`TYWRAP_PERF_BUDGETS`. -# Performance tuning -export TYWRAP_CACHE_DIR="./.tywrap/cache" -export TYWRAP_MEMORY_LIMIT="1024" -export TYWRAP_PERF_BUDGETS="1" -export TYWRAP_PERF_TIME_BUDGET_MS="2000" -export TYWRAP_PERF_MEMORY_BUDGET_MB="64" -export TYWRAP_CODEC_PERF_ITERATIONS="200" -export TYWRAP_CODEC_PERF_TIME_BUDGET_MS="500" -export TYWRAP_CODEC_PERF_MEMORY_BUDGET_MB="32" +Python executable and virtual environment selection are not configured through +environment variables today. Set them in `tywrap.config.*` or on the bridge: -# Development -export TYWRAP_VERBOSE="true" -export TYWRAP_HOT_RELOAD="true" +```ts +import { defineConfig } from 'tywrap'; + +export default defineConfig({ + runtime: { + node: { + pythonPath: '/usr/local/bin/python3', + virtualEnv: './venv', + timeout: 30000, + }, + }, +}); ``` +See [Environment Variables](/reference/env-vars) for the full implemented list. + ## Advanced Configuration Patterns ### Multi-Environment Setup @@ -1358,19 +1365,30 @@ try { ### Debugging Configuration -```json -{ - "runtime": { - "node": { - "timeoutMs": 0, // Disable timeout for debugging - "env": { - "PYTHONUNBUFFERED": "1", // Immediate stdout/stderr - "TYWRAP_DEBUG": "1" // Enable debug logging - } - } +Use the CLI's debug flag when you are troubleshooting wrapper generation: + +```bash +npx tywrap generate --debug +``` + +Use runtime log env vars for subprocess diagnostics: + +```bash +export TYWRAP_LOG_LEVEL=DEBUG +export TYWRAP_LOG_JSON=1 +``` + +If you need to disable timeouts or pass extra Python env vars while debugging, +do it on the bridge instance: + +```typescript +const bridge = new NodeBridge({ + pythonPath: 'python3', + timeoutMs: 0, + env: { + PYTHONUNBUFFERED: '1', }, - "debug": true -} +}); ``` ### Common Error Scenarios @@ -1427,17 +1445,16 @@ const [sin1, sin2, sin3] = await Promise.all([ ### Memory Management -```json -{ - "runtime": { - "node": { - "env": { - "PYTHONMALLOC": "malloc", // Use system malloc - "OMP_NUM_THREADS": "4" // Limit OpenMP threads - } - } - } -} +Pass Python-specific tuning through `env` on the bridge: + +```typescript +const bridge = new NodeBridge({ + pythonPath: 'python3', + env: { + PYTHONMALLOC: 'malloc', + OMP_NUM_THREADS: '4', + }, +}); ``` ## Production Deployment @@ -1505,8 +1522,8 @@ services: app: build: . environment: - - TYWRAP_PYTHON_PATH=/usr/bin/python3 - TYWRAP_CODEC_FALLBACK=json # For smaller containers + - TYWRAP_LOG_LEVEL=INFO ports: - '3000:3000' ``` @@ -1516,32 +1533,36 @@ services: ```bash # Production environment export NODE_ENV=production -export TYWRAP_PYTHON_PATH="/usr/local/bin/python3" -export TYWRAP_CACHE_DIR="/tmp/tywrap-cache" -export TYWRAP_MEMORY_LIMIT="2048" +export TYWRAP_CODEC_MAX_BYTES=10485760 +export TYWRAP_REQUEST_MAX_BYTES=1048576 +export TYWRAP_LOG_LEVEL=INFO # Security export PYTHONDONTWRITEBYTECODE=1 export PYTHONUNBUFFERED=1 ``` +Set the Python executable in config or when you construct the bridge: + +```typescript +const bridge = new NodeBridge({ + pythonPath: '/usr/local/bin/python3', +}); +``` + ## Security Considerations ### Subprocess Security -```json -{ - "runtime": { - "node": { - "cwd": "/safe/directory", // Restrict working directory - "env": { - "PATH": "/usr/bin:/bin", // Limit PATH - "PYTHONPATH": "/safe/python/libs" - }, - "timeoutMs": 10000 // Prevent hanging processes - } - } -} +```typescript +const bridge = new NodeBridge({ + cwd: '/safe/directory', + timeoutMs: 10000, + env: { + PATH: '/usr/bin:/bin', + PYTHONPATH: '/safe/python/libs', + }, +}); ``` ### Input Validation @@ -1604,25 +1625,25 @@ chmod +x /usr/local/bin/python3 **"Process timeout"**: -```json -{ - "runtime": { - "node": { - "timeoutMs": 60000, // Increase timeout - "env": { - "OMP_NUM_THREADS": "1" // Reduce parallelism - } - } - } -} +```typescript +const bridge = new NodeBridge({ + pythonPath: 'python3', + timeoutMs: 60000, + env: { + OMP_NUM_THREADS: '1', + }, +}); ``` ### Debug Mode ```bash -# Enable debug logging -export TYWRAP_DEBUG=1 -export TYWRAP_VERBOSE=1 +# Wrapper-generation diagnostics +npx tywrap generate --debug + +# Runtime bridge diagnostics +export TYWRAP_LOG_LEVEL=DEBUG +export TYWRAP_LOG_JSON=1 # Run with debug output node --trace-warnings your-app.js @@ -1659,14 +1680,11 @@ if __name__ == '__main__': pass ``` -```json -{ - "runtime": { - "node": { - "scriptPath": "./custom_bridge.py" - } - } -} +```typescript +const bridge = new NodeBridge({ + pythonPath: 'python3', + scriptPath: './custom_bridge.py', +}); ``` ### Process Pooling @@ -3010,9 +3028,8 @@ pip list ``` ```bash -# Enable debug logging -export TYWRAP_DEBUG=1 -npx tywrap generate +# Enable generator diagnostics +npx tywrap generate --debug ``` --- @@ -3023,18 +3040,17 @@ npx tywrap generate **Error**: `Python call timed out` or hanging operations **Solutions**: -```json -{ - "runtime": { - "node": { - "timeoutMs": 60000, // Increase timeout - "env": { - "OMP_NUM_THREADS": "1", // Reduce parallelism - "MKL_NUM_THREADS": "1" - } - } - } -} +```typescript +import { NodeBridge } from 'tywrap/node'; + +const bridge = new NodeBridge({ + pythonPath: 'python3', + timeoutMs: 60000, + env: { + OMP_NUM_THREADS: '1', + MKL_NUM_THREADS: '1', + }, +}); ``` ```typescript @@ -3068,22 +3084,29 @@ try { **Error**: Out of memory errors or process crashes **Solutions**: -```json -{ - "runtime": { - "node": { - "env": { - "NODE_OPTIONS": "--max-old-space-size=4096" - } - } - } -} +Large payloads and highly parallel native libraries are the usual causes. Tune +the bridge and the host process separately: + +```typescript +import { NodeBridge } from 'tywrap/node'; + +const bridge = new NodeBridge({ + pythonPath: 'python3', + env: { + OMP_NUM_THREADS: '1', + MKL_NUM_THREADS: '1', + }, +}); ``` ```bash -# Monitor memory usage +# Increase the Node.js heap when your app process is the bottleneck node --max-old-space-size=4096 your-app.js +# Cap bridge payload size so one call cannot blow up memory usage +export TYWRAP_CODEC_MAX_BYTES=10485760 +export TYWRAP_REQUEST_MAX_BYTES=1048576 + # Check system memory free -h # Linux top # General @@ -3125,10 +3148,8 @@ rm -rf generated/ rm -rf .tywrap/ npx tywrap generate -# Enable debug mode -export TYWRAP_DEBUG=1 -export TYWRAP_VERBOSE=1 -npx tywrap generate +# Enable generator diagnostics +npx tywrap generate --debug # Check Python module structure python3 -c " @@ -3260,38 +3281,38 @@ RUN apt-get update && apt-get install -y \ python3-dev \ && rm -rf /var/lib/apt/lists/* -# Set Python path -ENV TYWRAP_PYTHON_PATH=/usr/bin/python3 +# Keep Python subprocess output predictable +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 ``` +If Python is not discoverable on `PATH` in your image, pass +`pythonPath: '/usr/bin/python3'` when constructing `NodeBridge`. + --- ## Debug Mode and Logging ### Enable Debug Mode ```bash -# Environment variables -export TYWRAP_DEBUG=1 -export TYWRAP_VERBOSE=1 -export NODE_DEBUG=tywrap +# Generator diagnostics +npx tywrap generate --debug -# Run with debug output -npx tywrap generate 2>&1 | tee debug.log +# Runtime bridge diagnostics +TYWRAP_LOG_LEVEL=DEBUG TYWRAP_LOG_JSON=1 node app.js 2>tywrap.log + +# Extra Node.js warning detail +node --trace-warnings app.js ``` -### Custom Logging +### Bridge Diagnostics ```typescript -// Add logging to your application -import { createLogger } from 'tywrap/utils'; +import { NodeBridge } from 'tywrap/node'; -const logger = createLogger({ - level: 'debug', - output: './logs/tywrap.log' -}); +const bridge = new NodeBridge({ pythonPath: 'python3' }); -// Monitor bridge communication -bridge.on('request', (req) => logger.debug('Request:', req)); -bridge.on('response', (res) => logger.debug('Response:', res)); +const info = await bridge.getBridgeInfo({ refresh: true }); +console.error(info); ``` --- diff --git a/docs/troubleshooting/index.md b/docs/troubleshooting/index.md index 7e2380a..1186a0a 100644 --- a/docs/troubleshooting/index.md +++ b/docs/troubleshooting/index.md @@ -234,9 +234,8 @@ pip list ``` ```bash -# Enable debug logging -export TYWRAP_DEBUG=1 -npx tywrap generate +# Enable generator diagnostics +npx tywrap generate --debug ``` --- @@ -247,18 +246,17 @@ npx tywrap generate **Error**: `Python call timed out` or hanging operations **Solutions**: -```json -{ - "runtime": { - "node": { - "timeoutMs": 60000, // Increase timeout - "env": { - "OMP_NUM_THREADS": "1", // Reduce parallelism - "MKL_NUM_THREADS": "1" - } - } - } -} +```typescript +import { NodeBridge } from 'tywrap/node'; + +const bridge = new NodeBridge({ + pythonPath: 'python3', + timeoutMs: 60000, + env: { + OMP_NUM_THREADS: '1', + MKL_NUM_THREADS: '1', + }, +}); ``` ```typescript @@ -292,22 +290,29 @@ try { **Error**: Out of memory errors or process crashes **Solutions**: -```json -{ - "runtime": { - "node": { - "env": { - "NODE_OPTIONS": "--max-old-space-size=4096" - } - } - } -} +Large payloads and highly parallel native libraries are the usual causes. Tune +the bridge and the host process separately: + +```typescript +import { NodeBridge } from 'tywrap/node'; + +const bridge = new NodeBridge({ + pythonPath: 'python3', + env: { + OMP_NUM_THREADS: '1', + MKL_NUM_THREADS: '1', + }, +}); ``` ```bash -# Monitor memory usage +# Increase the Node.js heap when your app process is the bottleneck node --max-old-space-size=4096 your-app.js +# Cap bridge payload size so one call cannot blow up memory usage +export TYWRAP_CODEC_MAX_BYTES=10485760 +export TYWRAP_REQUEST_MAX_BYTES=1048576 + # Check system memory free -h # Linux top # General @@ -349,10 +354,8 @@ rm -rf generated/ rm -rf .tywrap/ npx tywrap generate -# Enable debug mode -export TYWRAP_DEBUG=1 -export TYWRAP_VERBOSE=1 -npx tywrap generate +# Enable generator diagnostics +npx tywrap generate --debug # Check Python module structure python3 -c " @@ -484,38 +487,38 @@ RUN apt-get update && apt-get install -y \ python3-dev \ && rm -rf /var/lib/apt/lists/* -# Set Python path -ENV TYWRAP_PYTHON_PATH=/usr/bin/python3 +# Keep Python subprocess output predictable +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 ``` +If Python is not discoverable on `PATH` in your image, pass +`pythonPath: '/usr/bin/python3'` when constructing `NodeBridge`. + --- ## Debug Mode and Logging ### Enable Debug Mode ```bash -# Environment variables -export TYWRAP_DEBUG=1 -export TYWRAP_VERBOSE=1 -export NODE_DEBUG=tywrap +# Generator diagnostics +npx tywrap generate --debug + +# Runtime bridge diagnostics +TYWRAP_LOG_LEVEL=DEBUG TYWRAP_LOG_JSON=1 node app.js 2>tywrap.log -# Run with debug output -npx tywrap generate 2>&1 | tee debug.log +# Extra Node.js warning detail +node --trace-warnings app.js ``` -### Custom Logging +### Bridge Diagnostics ```typescript -// Add logging to your application -import { createLogger } from 'tywrap/utils'; +import { NodeBridge } from 'tywrap/node'; -const logger = createLogger({ - level: 'debug', - output: './logs/tywrap.log' -}); +const bridge = new NodeBridge({ pythonPath: 'python3' }); -// Monitor bridge communication -bridge.on('request', (req) => logger.debug('Request:', req)); -bridge.on('response', (res) => logger.debug('Response:', res)); +const info = await bridge.getBridgeInfo({ refresh: true }); +console.error(info); ``` --- diff --git a/src/runtime/process-io.ts b/src/runtime/process-io.ts index e1c0295..8a022b9 100644 --- a/src/runtime/process-io.ts +++ b/src/runtime/process-io.ts @@ -21,6 +21,7 @@ import { BridgeTimeoutError, BridgeExecutionError, } from './errors.js'; +import { TimedOutRequestTracker } from './timed-out-request-tracker.js'; import type { Transport } from './transport.js'; // ============================================================================= @@ -36,6 +37,9 @@ const MAX_STDERR_BYTES = 8 * 1024; /** Default write queue timeout: 30 seconds */ const DEFAULT_WRITE_QUEUE_TIMEOUT_MS = 30_000; +/** Track timed-out/cancelled request IDs long enough to ignore late responses. */ +const TIMED_OUT_REQUEST_TTL_MS = 10 * 60 * 1000; + /** Regex for ANSI escape sequences */ const ANSI_ESCAPE_RE = /\u001b\[[0-9;]*[A-Za-z]/g; @@ -179,6 +183,9 @@ export class ProcessIO extends BoundedContext implements Transport { // Request tracking private readonly pending = new Map(); + private readonly timedOutRequests = new TimedOutRequestTracker({ + ttlMs: TIMED_OUT_REQUEST_TTL_MS, + }); private requestCount = 0; private needsRestart = false; @@ -260,6 +267,7 @@ export class ProcessIO extends BoundedContext implements Transport { if (timeoutMs > 0) { timer = setTimeout(() => { this.pending.delete(messageId); + this.timedOutRequests.mark(messageId); const stderrTail = this.getStderrTail(); const baseMsg = `Operation timed out after ${timeoutMs}ms`; const msg = stderrTail ? `${baseMsg}. Recent stderr:\n${stderrTail}` : baseMsg; @@ -273,6 +281,7 @@ export class ProcessIO extends BoundedContext implements Transport { clearTimeout(timer); } this.pending.delete(messageId); + this.timedOutRequests.mark(messageId); reject(new BridgeTimeoutError('Operation aborted')); }; @@ -359,6 +368,7 @@ export class ProcessIO extends BoundedContext implements Transport { // Clear buffers this.stdoutBuffer = ''; this.stderrBuffer = ''; + this.timedOutRequests.clear(); this.requestCount = 0; } @@ -596,8 +606,12 @@ export class ProcessIO extends BoundedContext implements Transport { const pending = this.pending.get(messageId); if (!pending) { - // Response for unknown request - could be for a timed-out request - // Log but don't fail + // Ignore expected late responses from timed-out/cancelled requests. + if (this.timedOutRequests.consume(messageId)) { + return; + } + // Unknown IDs while requests are pending indicate protocol desync. + this.handleProtocolError(`Unexpected response id ${messageId}`, line); return; } diff --git a/src/tywrap.ts b/src/tywrap.ts index c7b0d1f..693e5ef 100644 --- a/src/tywrap.ts +++ b/src/tywrap.ts @@ -147,7 +147,7 @@ export async function generate( const cacheKey = await computeCacheKey(moduleKey, resolvedOptions); let ir: unknown | null = null; let irError: string | undefined; - if (caching && fsUtils.isAvailable()) { + if (caching && fsUtils.isAvailable() && !checkMode) { try { const cached = await fsUtils.readFile(pathUtils.join(cacheDir, cacheKey)); ir = JSON.parse(cached); diff --git a/test/integration.test.ts b/test/integration.test.ts index bf1b3dd..4c68fd9 100644 --- a/test/integration.test.ts +++ b/test/integration.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; import { mkdtemp, rm, writeFile, mkdir } from 'node:fs/promises'; -import { join } from 'node:path'; +import { delimiter, join } from 'node:path'; import { tmpdir } from 'node:os'; import { generate } from '../src/tywrap.js'; import { processUtils, fsUtils } from '../src/utils/runtime.js'; @@ -80,6 +80,59 @@ describe('IR-only integration', () => { } }, 30_000); + it('check mode bypasses stale IR cache when caching is enabled', async () => { + const tempDir = await mkdtemp(join(tmpdir(), 'tywrap-check-cache-')); + const originalCwd = process.cwd(); + const originalPythonPath = process.env.PYTHONPATH; + try { + const importDir = join(tempDir, 'py'); + await mkdir(importDir, { recursive: true }); + const modulePath = join(importDir, 'local_stale_cache.py'); + await writeFile(modulePath, `def stable_value() -> int:\n return 1\n`, 'utf-8'); + + process.chdir(tempDir); + const repoTywrapIr = join(originalCwd, 'tywrap_ir'); + process.env.PYTHONPATH = originalPythonPath + ? `${repoTywrapIr}${delimiter}${originalPythonPath}` + : repoTywrapIr; + + const options = { + pythonModules: { local_stale_cache: { runtime: 'node', typeHints: 'strict' as const } }, + pythonImportPath: [importDir], + output: { + dir: join(tempDir, 'generated'), + format: 'esm' as const, + declaration: false, + sourceMap: false, + }, + runtime: { node: { pythonPath: defaultPythonPath } }, + performance: { caching: true, batching: false, compression: 'none' as const }, + development: { hotReload: false, sourceMap: false, validation: 'none' as const }, + }; + + const firstRun = await generate(options as any); + const generatedTsPath = firstRun.written.find(p => p.endsWith('.generated.ts')); + expect(typeof generatedTsPath).toBe('string'); + + await writeFile( + modulePath, + `def stable_value() -> int:\n return 1\n\ndef added_after_first_run() -> int:\n return 2\n`, + 'utf-8' + ); + + const checkResult = await generate(options as any, { check: true }); + expect((checkResult.outOfDate ?? []).includes(generatedTsPath as string)).toBe(true); + } finally { + if (originalPythonPath === undefined) { + delete process.env.PYTHONPATH; + } else { + process.env.PYTHONPATH = originalPythonPath; + } + process.chdir(originalCwd); + await rm(tempDir, { recursive: true, force: true }); + } + }, 30_000); + it('generate() emits valid TypeScript for advanced typing fixtures', async () => { const tempDir = await mkdtemp(join(process.cwd(), '.tmp-advanced-types-')); try { diff --git a/test/transport.test.ts b/test/transport.test.ts index 4c86ba4..45270d4 100644 --- a/test/transport.test.ts +++ b/test/transport.test.ts @@ -638,6 +638,48 @@ describe('ProcessIO', () => { await expect(pending).resolves.toContain('"id":0'); }); + it('rejects pending requests when an unexpected response id arrives', async () => { + const transport = new ProcessIO({ bridgeScript: '/path/to/bridge.py' }); + + const internals = transport as unknown as ProcessIOInternals; + internals._state = 'ready'; + internals.processExited = false; + internals.process = { + stdin: { + write: (): boolean => true, + }, + }; + + const messageId = 401; + const pending = transport.send(JSON.stringify(createValidMessage({ id: messageId })), 50); + + internals.handleResponseLine(JSON.stringify({ id: 999, result: 'wrong id' })); + + await expect(pending).rejects.toThrow(BridgeProtocolError); + await expect(pending).rejects.toThrow(/Unexpected response id 999/); + }); + + it('ignores late responses for requests that already timed out', async () => { + const transport = new ProcessIO({ bridgeScript: '/path/to/bridge.py' }); + + const internals = transport as unknown as ProcessIOInternals; + internals._state = 'ready'; + internals.processExited = false; + internals.process = { + stdin: { + write: (): boolean => true, + }, + }; + + const messageId = 402; + const pending = transport.send(JSON.stringify(createValidMessage({ id: messageId })), 10); + + await expect(pending).rejects.toThrow(BridgeTimeoutError); + expect(() => + internals.handleResponseLine(JSON.stringify({ id: messageId, result: 'late result' })) + ).not.toThrow(BridgeProtocolError); + }); + it('includes stderr diagnostics when stdin write fails', async () => { const transport = new ProcessIO({ bridgeScript: '/path/to/bridge.py' }); diff --git a/tywrap_ir/tests/test_ir_async_generator_flags.py b/tywrap_ir/tests/test_ir_async_generator_flags.py new file mode 100644 index 0000000..7854ae1 --- /dev/null +++ b/tywrap_ir/tests/test_ir_async_generator_flags.py @@ -0,0 +1,40 @@ +import unittest +from collections.abc import AsyncIterator, Iterator + +from tywrap_ir.ir import _extract_function + + +async def sample_async_generator() -> AsyncIterator[int]: + yield 1 + + +async def sample_async_coroutine() -> int: + return 1 + + +def sample_sync_generator() -> Iterator[int]: + yield 1 + + +class IRFunctionFlagTests(unittest.TestCase): + def test_async_generator_is_marked_as_generator(self) -> None: + result = _extract_function(sample_async_generator, "tests.sample_async_generator") + assert result is not None + assert result.is_async + assert result.is_generator + + def test_async_coroutine_is_not_marked_as_generator(self) -> None: + result = _extract_function(sample_async_coroutine, "tests.sample_async_coroutine") + assert result is not None + assert result.is_async + assert not result.is_generator + + def test_sync_generator_flags(self) -> None: + result = _extract_function(sample_sync_generator, "tests.sample_sync_generator") + assert result is not None + assert not result.is_async + assert result.is_generator + + +if __name__ == "__main__": + unittest.main() diff --git a/tywrap_ir/tests/test_optimized_ir_options.py b/tywrap_ir/tests/test_optimized_ir_options.py new file mode 100644 index 0000000..e71de1d --- /dev/null +++ b/tywrap_ir/tests/test_optimized_ir_options.py @@ -0,0 +1,41 @@ +import unittest + +from tywrap_ir.optimized_ir import ( + _global_cache, + _global_extractor, + extract_module_ir_optimized, +) + + +class OptimizedIRExtractorOptionTests(unittest.TestCase): + def setUp(self) -> None: + _global_extractor.enable_caching = True + _global_extractor.enable_parallel = True + _global_extractor._cache = _global_cache + _global_extractor.clear_cache() + + def tearDown(self) -> None: + _global_extractor.enable_caching = True + _global_extractor.enable_parallel = True + _global_extractor._cache = _global_cache + _global_extractor.clear_cache() + + def test_disabling_options_does_not_mutate_global_extractor(self) -> None: + extract_module_ir_optimized("math", enable_caching=False, enable_parallel=False) + + self.assertTrue(_global_extractor.enable_caching) + self.assertTrue(_global_extractor.enable_parallel) + self.assertIs(_global_extractor._cache, _global_cache) + + def test_default_path_still_uses_shared_cache(self) -> None: + extract_module_ir_optimized("math", enable_caching=True, enable_parallel=True) + hits_before = _global_extractor.get_stats()["cache"]["hits"] + + extract_module_ir_optimized("math", enable_caching=True, enable_parallel=True) + hits_after = _global_extractor.get_stats()["cache"]["hits"] + + self.assertGreaterEqual(hits_after, hits_before + 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tywrap_ir/tests/test_optimized_ir_parallel_warnings.py b/tywrap_ir/tests/test_optimized_ir_parallel_warnings.py new file mode 100644 index 0000000..3cfaf80 --- /dev/null +++ b/tywrap_ir/tests/test_optimized_ir_parallel_warnings.py @@ -0,0 +1,27 @@ +import importlib +import unittest + +from tywrap_ir.optimized_ir import OptimizedIRExtractor + + +class OptimizedIRParallelWarningTests(unittest.TestCase): + def test_parallel_extraction_reports_component_failures_in_warnings(self) -> None: + extractor = OptimizedIRExtractor(enable_caching=False, enable_parallel=True, max_workers=2) + module = importlib.import_module("math") + + extractor._extract_functions_optimized = lambda *_args, **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] + RuntimeError("boom") + ) + extractor._extract_classes_optimized = lambda *_args, **_kwargs: [] # type: ignore[method-assign] + extractor._extract_constants_optimized = lambda *_args, **_kwargs: [] # type: ignore[method-assign] + extractor._extract_type_aliases_optimized = lambda *_args, **_kwargs: [] # type: ignore[method-assign] + + result = extractor._extract_parallel(module, "math", "0.1.0", False) + + self.assertEqual(result["module"], "math") + self.assertIn("warnings", result) + self.assertTrue(any("Error extracting functions from math: boom" in w for w in result["warnings"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tywrap_ir/tywrap_ir/ir.py b/tywrap_ir/tywrap_ir/ir.py index c75f3dd..7ffca57 100644 --- a/tywrap_ir/tywrap_ir/ir.py +++ b/tywrap_ir/tywrap_ir/ir.py @@ -232,7 +232,9 @@ def _extract_function(obj: Any, qualname: str) -> Optional[IRFunction]: returns = hints.get("return", sig.return_annotation) is_async = inspect.iscoroutinefunction(obj) or inspect.isasyncgenfunction(obj) - is_generator = inspect.isgeneratorfunction(obj) + # Async generators must be marked as generators so callers can distinguish + # them from plain coroutines. + is_generator = inspect.isgeneratorfunction(obj) or inspect.isasyncgenfunction(obj) return IRFunction( name=getattr(obj, "__name__", qualname.split(".")[-1]), diff --git a/tywrap_ir/tywrap_ir/optimized_ir.py b/tywrap_ir/tywrap_ir/optimized_ir.py index 781e3ea..7999ad9 100644 --- a/tywrap_ir/tywrap_ir/optimized_ir.py +++ b/tywrap_ir/tywrap_ir/optimized_ir.py @@ -205,7 +205,8 @@ def _extract_parallel(self, ir_version: str, include_private: bool) -> Dict[str, Any]: """Extract IR components in parallel""" - + + warnings: List[str] = [] with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor: # Submit all extraction tasks futures = { @@ -222,10 +223,14 @@ def _extract_parallel(self, try: results[key] = future.result(timeout=30) # 30 second timeout except concurrent.futures.TimeoutError: - print(f"⚠️ Timeout extracting {key} from {module_name}", file=sys.stderr) + warning = f"Timeout extracting {key} from {module_name}" + print(f"⚠️ {warning}", file=sys.stderr) + warnings.append(warning) results[key] = [] if key != 'metadata' else {} except Exception as e: - print(f"❌ Error extracting {key} from {module_name}: {e}", file=sys.stderr) + warning = f"Error extracting {key} from {module_name}: {e}" + print(f"❌ {warning}", file=sys.stderr) + warnings.append(warning) results[key] = [] if key != 'metadata' else {} # Build IR module @@ -237,7 +242,7 @@ def _extract_parallel(self, constants=results.get('constants', []), type_aliases=results.get('type_aliases', []), metadata=results.get('metadata', {}), - warnings=[] # TODO: Collect warnings from parallel execution + warnings=warnings ) # Update statistics @@ -423,16 +428,19 @@ def extract_module_ir_optimized(module_name: str, Returns: Dictionary containing the IR module data """ - - # Configure extractor if needed - if not enable_caching and _global_extractor.enable_caching: - _global_extractor._cache = None - _global_extractor.enable_caching = False - - if not enable_parallel and _global_extractor.enable_parallel: - _global_extractor.enable_parallel = False - - return _global_extractor.extract_module_ir_optimized( + + # Preserve the shared optimized extractor for the default fast path. + # Non-default option combinations should not mutate global state for later calls. + if enable_caching and enable_parallel: + extractor = _global_extractor + else: + extractor = OptimizedIRExtractor( + enable_caching=enable_caching, + enable_parallel=enable_parallel, + max_workers=_global_extractor.max_workers, + ) + + return extractor.extract_module_ir_optimized( module_name, ir_version=ir_version, include_private=include_private @@ -541,4 +549,4 @@ def benchmark_ir_extraction(module_names: List[str], print(f" {module}: {data['avg_time']:.2f}ms avg, {data['speedup']:.2f}x speedup", file=sys.stderr) # Output results as JSON for analysis - print(json.dumps(optimized_results, indent=2)) \ No newline at end of file + print(json.dumps(optimized_results, indent=2))