From 6c2c8c232fd5b5aaa21de1a539b6b179ad7f3f94 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Fri, 3 Oct 2025 08:31:14 -0400 Subject: [PATCH 01/26] added new sig route 2 --- .cursor/rules/code-generation.mdc | 18 +++++++--- build-and-copy.sh | 45 ++++++++++++++++++++++++ src/apis/prompt/generated/index.d.ts | 23 ++++++++++--- src/apis/prompt/generated/index.js | 5 +++ src/apis/prompt/generated/index.ts | 11 ------ src/apis/prompt/generic_types.ts | 49 ++++++++++++++++++++++++++ src/apis/prompt/index.ts | 23 +++++++++++-- src/apis/prompt/signature.ts | 51 ++++++++++++++++++++++++++++ src/server/server.ts | 35 +++++++++++++++++++ src/types.ts | 18 +++++++++- 10 files changed, 254 insertions(+), 24 deletions(-) create mode 100755 build-and-copy.sh create mode 100644 src/apis/prompt/generated/index.js delete mode 100644 src/apis/prompt/generated/index.ts create mode 100644 src/apis/prompt/generic_types.ts create mode 100644 src/apis/prompt/signature.ts diff --git a/.cursor/rules/code-generation.mdc b/.cursor/rules/code-generation.mdc index a627d51f..78a47bbc 100644 --- a/.cursor/rules/code-generation.mdc +++ b/.cursor/rules/code-generation.mdc @@ -155,11 +155,21 @@ try { 2. **Update SDK to handle new structure** in `src/apis/prompt/index.ts` 3. **Bump SDK version** in `package.json` 4. **Build SDK**: `npm run build` -5. **Install in test project**: `npm install /path/to/sdk-js` -6. **Run bundle command**: `agentuity-cli bundle` to regenerate content -7. **Copy dist to node_modules**: `cp -r dist/* node_modules/@agentuity/sdk/dist/` (if needed) +6. **Run bundle command**: `go run . bundle` to regenerate content +8. never run `go run . dev` for me +7. **Copy dist to node_modules**: Use `./build-and-copy.sh ` or `cp -r dist/* node_modules/@agentuity/sdk/dist/` (if needed) + +**Note**: After the first setup, you only need to run step 7 for subsequent changes to avoid reinstalling the entire package. + +### Quick SDK Update Workflow: +For faster iteration when testing SDK changes: +```bash +# In sdk-js directory +npm run build +./build-and-copy.sh /path/to/test-project/node_modules/@agentuity/sdk/dist +``` -**Note**: After the first setup, you only need to run step 7 (`cp -r dist/* node_modules/@agentuity/sdk/dist/`) for subsequent changes to avoid reinstalling the entire package. +This copies the built SDK directly to your test project without reinstalling the package. ### Required Exports - โœ… Always export `interpolateTemplate` from main SDK index diff --git a/build-and-copy.sh b/build-and-copy.sh new file mode 100755 index 00000000..088c1900 --- /dev/null +++ b/build-and-copy.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Build and copy script for @agentuity/sdk +# Usage: ./build-and-copy.sh + +set -e # Exit on any error + +# Check if target directory is provided +if [ $# -eq 0 ]; then + echo "Usage: $0 " + echo "Example: $0 /path/to/your/project/node_modules/@agentuity/sdk/dist" + exit 1 +fi + +TARGET_DIR="$1" + +echo "๐Ÿ”จ Building @agentuity/sdk in current directory..." + +# Clean and build +npm run build + +# Check if build was successful +if [ ! -d "dist" ]; then + echo "โŒ Build failed - dist directory not found" + exit 1 +fi + +echo "โœ… Build completed successfully" + +# Create target directory if it doesn't exist +echo "๐Ÿ“ Creating target directory: $TARGET_DIR" +mkdir -p "$TARGET_DIR" + +# Copy dist contents to target directory +echo "๐Ÿ“ฆ Copying dist/* to $TARGET_DIR" +cp -r dist/* "$TARGET_DIR/" + +# Copy package.json and LICENSE.md +echo "๐Ÿ“„ Copying package.json and LICENSE.md" +cp package.json "$TARGET_DIR/" +cp LICENSE.md "$TARGET_DIR/" + +echo "โœ… Successfully copied @agentuity/sdk to $TARGET_DIR" +echo "๐Ÿ“Š Contents copied:" +ls -la "$TARGET_DIR" diff --git a/src/apis/prompt/generated/index.d.ts b/src/apis/prompt/generated/index.d.ts index 41db119a..46e675d5 100644 --- a/src/apis/prompt/generated/index.d.ts +++ b/src/apis/prompt/generated/index.d.ts @@ -1,5 +1,18 @@ -// biome-ignore lint/suspicious/noEmptyInterface: -export interface PromptsCollection {} -export declare const prompts: PromptsCollection; -export type PromptConfig = any; -export type PromptName = any; +// This file will be replaced by the CLI with actual generated types +// Do not edit this file manually - it will be overwritten during the bundle process + +import type { Prompt } from '../generic_types.js'; + +// Placeholder interface that will be replaced with actual generated prompts +export interface GeneratedPromptsCollection { + [promptSlug: string]: any; +} + +// Placeholder signature collection that will be replaced +export type SignaturesCollection = { + [promptSlug: string]: (params: any) => string; +}; + +// Placeholder exports that will be replaced +export const prompts: GeneratedPromptsCollection = {} as any; +export const signatures: SignaturesCollection = {} as any; diff --git a/src/apis/prompt/generated/index.js b/src/apis/prompt/generated/index.js new file mode 100644 index 00000000..7e238212 --- /dev/null +++ b/src/apis/prompt/generated/index.js @@ -0,0 +1,5 @@ +// This file will be replaced by the CLI with actual generated JavaScript +// Do not edit this file manually - it will be overwritten during the bundle process + +export const prompts = {}; +export const signatures = {}; diff --git a/src/apis/prompt/generated/index.ts b/src/apis/prompt/generated/index.ts deleted file mode 100644 index c7394aa4..00000000 --- a/src/apis/prompt/generated/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -export type PromptConfig = any; -export type PromptName = any; - -// This will be replaced by the CLI with the actual generated types -export interface PromptsCollection { - [promptSlug: string]: { - slug: string; - system: { compile: (variables?: Record) => string }; - prompt: { compile: (variables?: Record) => string }; - }; -} diff --git a/src/apis/prompt/generic_types.ts b/src/apis/prompt/generic_types.ts new file mode 100644 index 00000000..5999ad52 --- /dev/null +++ b/src/apis/prompt/generic_types.ts @@ -0,0 +1,49 @@ +// Simplified generic types for prompt generation +type HasSystem = boolean; +type HasPrompt = boolean; +type HasVariables = boolean; + +// Base prompt structure +interface BasePrompt { + slug: string; +} + +// Conditional system field +interface SystemField { + system: (params: { system: Record }) => string; +} + +// Conditional prompt field +interface PromptField { + prompt: (params: { prompt: Record }) => string; +} + +// Simplified generic prompt type +export type Prompt< + THasSystem extends HasSystem = true, + THasPrompt extends HasPrompt = true, + THasVariables extends HasVariables = true, + TVariablesSystem = Record, + TVariablesPrompt = Record, +> = BasePrompt & + (THasSystem extends true ? SystemField : Record) & + (THasPrompt extends true ? PromptField : Record) & + (THasVariables extends true + ? { + variables: { + system?: TVariablesSystem; + prompt?: TVariablesPrompt; + }; + } + : Record); + +// Simple signature function type +export type PromptSignature> = ( + params: any +) => string; + +// Simple collection types +export type PromptsCollection = Record; +export type GetPromptSignatures> = { + [K in keyof T]: PromptSignature; +}; diff --git a/src/apis/prompt/index.ts b/src/apis/prompt/index.ts index 7f200414..3c9b4904 100644 --- a/src/apis/prompt/index.ts +++ b/src/apis/prompt/index.ts @@ -4,7 +4,11 @@ import fs from 'fs/promises'; import path from 'path'; import { pathToFileURL } from 'url'; import { PromptConfig, PromptName } from './generated/_index.js'; -import type { PromptsCollection } from './generated/index.js'; +import type { PromptsCollection } from './generic_types.js'; +import { + createPromptSignatures, + type GetPromptSignatures, +} from './signature.js'; // Default empty prompts object const defaultPrompts = {}; @@ -16,10 +20,12 @@ interface GeneratedModule { export default class PromptAPI { public prompts: PromptsCollection; + public signatures: GetPromptSignatures; constructor() { // Initialize with empty prompts by default this.prompts = defaultPrompts; + this.signatures = createPromptSignatures(this.prompts); } /** @@ -128,6 +134,8 @@ export default class PromptAPI { // ); this.prompts = generatedModule.prompts || (defaultPrompts as PromptsCollection); + // Update signatures when prompts are loaded + this.signatures = createPromptSignatures(this.prompts); // console.log('Final prompts:', Object.keys(this.prompts)); } catch (error) { // Fallback to empty prompts if generated file doesn't exist @@ -136,6 +144,8 @@ export default class PromptAPI { error instanceof Error ? error.message : String(error) ); this.prompts = defaultPrompts; + // Update signatures with empty prompts + this.signatures = createPromptSignatures(this.prompts); console.warn( 'โš ๏ธ No generated prompts found. Run `agentuity bundle` to generate prompts from src/prompts.yaml' ); @@ -144,5 +154,12 @@ export default class PromptAPI { } // Re-export generated types and prompts (following POC pattern) -export { defaultPrompts as prompts, PromptConfig, PromptName }; -export * from './generated/index.js'; +export { defaultPrompts, PromptConfig, PromptName }; +export { + GeneratedPromptsCollection, + prompts, + SignaturesCollection, + signatures, +} from './generated/index.js'; +export * from './generic_types.js'; +export { createPromptSignature, createPromptSignatures } from './signature.js'; diff --git a/src/apis/prompt/signature.ts b/src/apis/prompt/signature.ts new file mode 100644 index 00000000..a531cf77 --- /dev/null +++ b/src/apis/prompt/signature.ts @@ -0,0 +1,51 @@ +import type { Prompt, PromptSignature } from './generic_types.js'; + +/** + * Creates a signature function for a specific prompt type + * The function signature adapts based on whether the prompt has system, prompt, and variables + */ +export function createPromptSignature( + _prompt: T +): PromptSignature { + // This is a type-safe wrapper that will be replaced by the CLI with actual implementation + // The CLI will generate the appropriate function based on the prompt's structure + + // For now, return a generic function that handles all cases + // The CLI will replace this with the specific implementation + return ((..._args: unknown[]) => { + // This will be replaced by the CLI with the actual template compilation logic + throw new Error( + 'Generated signature function not found. Run `agentuity bundle` to generate prompts.' + ); + }) as unknown as PromptSignature; +} + +/** + * Type-safe prompt signature factory + * This function creates signature functions that match the exact structure of each prompt + */ +export function createPromptSignatures>( + prompts: T +): { + [K in keyof T]: PromptSignature; +} { + const signatures = {} as any; + + for (const [key, prompt] of Object.entries(prompts)) { + signatures[key] = createPromptSignature(prompt); + } + + return signatures; +} + +/** + * Utility type to extract the signature function type for a specific prompt + */ +export type GetPromptSignature = PromptSignature; + +/** + * Utility type to extract all signature functions from a prompts collection + */ +export type GetPromptSignatures> = { + [K in keyof T]: PromptSignature; +}; diff --git a/src/server/server.ts b/src/server/server.ts index 17618ec5..878d0e1b 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -218,6 +218,41 @@ export async function createServerContext( stream, email, _experimental_prompts: () => promptAPI.prompts, + prompts: { + getPrompt: (slug: T) => { + const prompt = promptAPI.prompts[slug]; + if (!prompt) { + throw new Error(`Prompt '${slug}' not found`); + } + return prompt; + }, + compile: ( + slug: T, + params: any + ) => { + const prompt = promptAPI.prompts[slug]; + if (!prompt) { + throw new Error(`Prompt '${slug}' not found`); + } + + // Use the signature function if available + const signature = promptAPI.signatures[slug]; + if (signature) { + return signature(params); + } + + // Fallback to manual compilation + let result = ''; + if (prompt.system && params.system) { + result += prompt.system(params); + } + if (prompt.prompt && params.prompt) { + if (result) result += '\n'; + result += prompt.prompt(params); + } + return result; + }, + }, discord, objectstore, patchportal, diff --git a/src/types.ts b/src/types.ts index 1f4a6429..344dfffa 100644 --- a/src/types.ts +++ b/src/types.ts @@ -928,7 +928,23 @@ export interface AgentContext { /** * get the prompts collection for compiling dynamic prompts */ - _experimental_prompts(): import('./apis/prompt/generated/index.js').PromptsCollection; + _experimental_prompts(): import('./apis/prompt/generated/index.js').GeneratedPromptsCollection; + + /** + * prompts API for accessing and compiling prompts + */ + prompts: { + getPrompt< + T extends + keyof import('./apis/prompt/generated/index.js').GeneratedPromptsCollection, + >( + slug: T + ): import('./apis/prompt/generated/index.js').GeneratedPromptsCollection[T]; + compile< + T extends + keyof import('./apis/prompt/generated/index.js').GeneratedPromptsCollection, + >(slug: T, params: any): string; + }; } /** From 637b9bc6003e2ed0d1791987f5c11f62dc5412e9 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Fri, 3 Oct 2025 09:00:21 -0400 Subject: [PATCH 02/26] ts seems good onto js --- .cursor/rules/code-generation.mdc | 202 ++++++++++++++++++++++++++---- src/server/server.ts | 28 +---- src/types.ts | 6 +- 3 files changed, 185 insertions(+), 51 deletions(-) diff --git a/.cursor/rules/code-generation.mdc b/.cursor/rules/code-generation.mdc index 78a47bbc..d9c42411 100644 --- a/.cursor/rules/code-generation.mdc +++ b/.cursor/rules/code-generation.mdc @@ -1,26 +1,129 @@ --- -description: Code Generation Development Rules for SDK -globs: src/apis/**/*.ts, src/server/*.ts, src/types.ts +description: Code Generation Development Rules for CLI and SDK +globs: internal/bundler/*.go, cmd/*.go, src/apis/**/*.ts, src/server/*.ts, src/types.ts alwaysApply: true --- # Code Generation Development Rules -> **โš ๏ธ IMPORTANT**: These rules work in conjunction with the CLI rules. When updating these SDK rules, also update `cli/.cursor/rules/code-generation.mdc` to keep them in sync. +> **โš ๏ธ IMPORTANT**: These rules apply to both CLI and SDK development. Keep both codebases in sync when making changes. ## Core Principles +### Never Modify Generated Content in Source Files +- โŒ NEVER edit files in `sdk-js/src/` that contain generated content +- โŒ NEVER hardcode generated content like `copyWriter` prompts in source files +- โœ… ALWAYS generate content into `node_modules/@agentuity/sdk/dist/` or `src/` directories +- โœ… Use dynamic loading patterns for generated content + +### Code Generation Workflow +1. **Modify CLI generation logic** (e.g., `internal/bundler/prompts.go`) +2. **Update SDK to handle generated content dynamically** (e.g., `src/apis/prompt/index.ts`) +3. **Build and test the full pipeline**: CLI generation โ†’ SDK loading โ†’ Agent usage + ### Optional Field Handling - โœ… Generated code should NEVER require optional chaining (`?.`) - โœ… Always generate both `system` and `prompt` fields, even if empty - โœ… Empty fields should return empty strings, not undefined - โŒ Never generate partial objects that require optional chaining -### Never Modify Generated Content in Source Files -- โŒ NEVER edit files in `src/` that contain generated content -- โŒ NEVER hardcode generated content like `copyWriter` prompts in source files -- โœ… ALWAYS load generated content dynamically at runtime -- โœ… Provide fallbacks for missing generated content +## Architecture Overview + +### File Structure +``` +sdk-js/src/apis/prompt/ +โ”œโ”€โ”€ generic_types.ts # Simple utility types for CLI to use +โ”œโ”€โ”€ generated/ +โ”‚ โ”œโ”€โ”€ index.d.ts # Shell TypeScript definitions (replaced by CLI) +โ”‚ โ”œโ”€โ”€ index.js # Shell JavaScript (replaced by CLI) +โ”‚ โ””โ”€โ”€ _index.js # Actual generated JavaScript (created by CLI) +โ”œโ”€โ”€ index.ts # Main API with dynamic loading +โ””โ”€โ”€ signature.ts # Signature function factory +``` + +### Key Learnings from Implementation + +#### 1. Simplified Architecture +- **Use shell files**: `index.d.ts` and `index.js` are placeholders that get completely replaced +- **Less complex generics**: Avoid overly complex TypeScript generics that cause compilation issues +- **Rely on code generation**: Let the CLI do the heavy lifting instead of complex type manipulation + +#### 2. Slug-Based Naming +- **Use slugs directly**: Generate code using the original slug names (e.g., `'simple-helper'`) +- **Quote property names**: Use `'slug-name'` syntax in TypeScript interfaces +- **Bracket notation**: Use `prompts['slug-name']` in JavaScript for property access +- **CamelCase variables**: Use `strcase.ToLowerCamel(slug)` for JavaScript variable names + +#### 3. Type Safety Without Complexity +```typescript +// โœ… Simple, working approach +export type PromptsCollection = Record; +export type PromptSignature = (params: any) => string; + +// โŒ Avoid overly complex generics that cause compilation issues +export type ComplexGeneric> = ... +``` + +## CLI-Specific Rules + +### Generation Target Locations +```go +// โœ… Correct: Generate into installed SDK +sdkPath := filepath.Join(root, "node_modules", "@agentuity", "sdk", "dist", "generated") + +// โŒ Wrong: Generate into source SDK +sdkPath := filepath.Join(root, "src", "generated") +``` + +### Slug Handling in Code Generation +```go +// โœ… Correct: Use slugs directly with proper quoting +const %s = { + slug: "%s", + // ... fields +};`, strcase.ToLowerCamel(prompt.Slug), prompt.Slug + +// In TypeScript interfaces +exports = append(exports, fmt.Sprintf(" '%s': %s;", prompt.Slug, strcase.ToCamel(prompt.Slug))) + +// In JavaScript property access +bodyParts = append(bodyParts, fmt.Sprintf("const result = prompts['%s'].system(params)", prompt.Slug)) +``` + +### File Generation Pattern +```go +func FindSDKGeneratedDir(ctx BundleContext, projectDir string) (string, error) { + possibleRoots := []string{ + findWorkspaceInstallDir(ctx.Logger, projectDir), + projectDir, + } + + for _, root := range possibleRoots { + // Try dist directory first (production) + sdkPath := filepath.Join(root, "node_modules", "@agentuity", "sdk", "dist", "generated") + if _, err := os.Stat(filepath.Join(root, "node_modules", "@agentuity", "sdk")); err == nil { + if err := os.MkdirAll(sdkPath, 0755); err == nil { + return sdkPath, nil + } + } + // Fallback to src directory (development) + sdkPath = filepath.Join(root, "node_modules", "@agentuity", "sdk", "src", "generated") + if _, err := os.Stat(filepath.Join(root, "node_modules", "@agentuity", "sdk", "src")); err == nil { + if err := os.MkdirAll(sdkPath, 0755); err == nil { + return sdkPath, nil + } + } + } + return "", fmt.Errorf("could not find @agentuity/sdk in node_modules") +} +``` + +## SDK-Specific Rules + +### Path Resolution +- Use absolute paths only (relative paths don't work in bundled environments) +- Check `dist/` directory first, then `src/` directory +- Always provide fallbacks for missing generated content ### Dynamic Loading Pattern Generated content doesn't exist at SDK build time, so use dynamic loading patterns: @@ -51,13 +154,6 @@ public async loadGeneratedContent(): Promise { } ``` -## SDK-Specific Rules - -### Path Resolution -- Use absolute paths only (relative paths don't work in bundled environments) -- Check `dist/` directory first, then `src/` directory -- Always provide fallbacks for missing generated content - ### Context Integration ```typescript // โœ… Good: Load generated content in context creation @@ -69,7 +165,17 @@ export async function createServerContext(req: ServerContextRequest): Promise promptAPI.prompts, + prompts: { + getPrompt: (slug: string) => promptAPI.prompts[slug], + compile: (slug: string, params: any) => { + // Use signature functions or fallback to manual compilation + const signature = promptAPI.signatures[slug]; + if (signature) { + return signature(params); + } + // Fallback logic... + } + }, }; } ``` @@ -78,6 +184,7 @@ export async function createServerContext(req: ServerContextRequest): Promise> = ... ``` ### โœ… Do This Instead +```go +// Generate dynamic content from YAML/data +content := GenerateTypeScriptTypes(prompts) + +// Generate to installed SDK +sdkPath := filepath.Join(root, "node_modules", "@agentuity", "sdk", "dist", "generated") + +// Check and create directories +if err := os.MkdirAll(sdkPath, 0755); err != nil { + return fmt.Errorf("failed to create directory: %w", err) +} + +// Use quoted slugs in TypeScript +'optional-with-defaults': OptionalWithDefaults; // โœ… Valid syntax +``` + ```typescript // Dynamic loading with absolute paths const possiblePaths = [ @@ -139,6 +279,10 @@ try { } catch (error) { this.content = defaultContent; } + +// Simple, working types +export type PromptsCollection = Record; +export type PromptSignature = (params: any) => string; ``` ## Build Considerations @@ -147,6 +291,7 @@ try { - Use `require()` for CommonJS compatibility in bundled environments - Avoid `import()` statements for generated content - Ensure fallbacks work when generated content is missing +- Keep TypeScript types simple to avoid compilation issues ## Development Workflow @@ -155,11 +300,10 @@ try { 2. **Update SDK to handle new structure** in `src/apis/prompt/index.ts` 3. **Bump SDK version** in `package.json` 4. **Build SDK**: `npm run build` -6. **Run bundle command**: `go run . bundle` to regenerate content -8. never run `go run . dev` for me -7. **Copy dist to node_modules**: Use `./build-and-copy.sh ` or `cp -r dist/* node_modules/@agentuity/sdk/dist/` (if needed) +5. **Run bundle command**: `go run . bundle` to regenerate content +6. **Copy dist to node_modules**: Use `./build-and-copy.sh ` or `cp -r dist/* node_modules/@agentuity/sdk/dist/` (if needed) -**Note**: After the first setup, you only need to run step 7 for subsequent changes to avoid reinstalling the entire package. +**Note**: After the first setup, you only need to run step 6 for subsequent changes to avoid reinstalling the entire package. ### Quick SDK Update Workflow: For faster iteration when testing SDK changes: @@ -176,4 +320,20 @@ This copies the built SDK directly to your test project without reinstalling the - โœ… Generated content must import from `@agentuity/sdk` (not relative paths) - โœ… Ensure all dependencies are properly exported -Remember: The SDK's job is to load generated content dynamically, not contain hardcoded generated content. \ No newline at end of file +## Key Implementation Notes + +### What We Learned +1. **Shell-based approach works better** than complex type manipulation +2. **Slug-based naming** is more intuitive than camelCase conversion +3. **Simple types** are more maintainable than complex generics +4. **Complete file replacement** is cleaner than partial updates +5. **Proper quoting** is essential for TypeScript interfaces with hyphens + +### Best Practices +- Use slugs directly in generated code with proper quoting +- Keep generic types simple and focused +- Rely on code generation for complex type structures +- Provide clear fallbacks for missing generated content +- Test the full pipeline regularly + +Remember: The CLI's job is to generate content into the installed SDK, and the SDK's job is to load generated content dynamically, not contain hardcoded generated content. \ No newline at end of file diff --git a/src/server/server.ts b/src/server/server.ts index 878d0e1b..61738929 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -226,32 +226,8 @@ export async function createServerContext( } return prompt; }, - compile: ( - slug: T, - params: any - ) => { - const prompt = promptAPI.prompts[slug]; - if (!prompt) { - throw new Error(`Prompt '${slug}' not found`); - } - - // Use the signature function if available - const signature = promptAPI.signatures[slug]; - if (signature) { - return signature(params); - } - - // Fallback to manual compilation - let result = ''; - if (prompt.system && params.system) { - result += prompt.system(params); - } - if (prompt.prompt && params.prompt) { - if (result) result += '\n'; - result += prompt.prompt(params); - } - return result; - }, + // Use the generated compile function with proper type safety + compile: promptAPI.prompts.compile, }, discord, objectstore, diff --git a/src/types.ts b/src/types.ts index 344dfffa..5039c635 100644 --- a/src/types.ts +++ b/src/types.ts @@ -940,10 +940,8 @@ export interface AgentContext { >( slug: T ): import('./apis/prompt/generated/index.js').GeneratedPromptsCollection[T]; - compile< - T extends - keyof import('./apis/prompt/generated/index.js').GeneratedPromptsCollection, - >(slug: T, params: any): string; + // Use the generated compile function with proper type safety + compile: import('./apis/prompt/generated/index.js').GeneratedPromptsCollection['compile']; }; } From a67a5e7f956261add579209244b57999dd3cedbc Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Fri, 3 Oct 2025 12:11:12 -0400 Subject: [PATCH 03/26] stuff is working --- package.json | 1 + build-and-copy.sh => scripts/build-and-copy.sh | 0 src/server/server.ts | 12 +----------- src/types.ts | 4 +--- 4 files changed, 3 insertions(+), 14 deletions(-) rename build-and-copy.sh => scripts/build-and-copy.sh (100%) diff --git a/package.json b/package.json index ac50b3c4..ff692d1b 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ }, "scripts": { "build": "rm -rf dist && tsup && npm run types", + "build-and-copy": "./scripts/build-and-copy.sh", "types": "tsc --emitDeclarationOnly --declaration", "bun:start": "npm run build && bun run ./dist/index.js", "node:start": "npm run build && node ./dist/index.js", diff --git a/build-and-copy.sh b/scripts/build-and-copy.sh similarity index 100% rename from build-and-copy.sh rename to scripts/build-and-copy.sh diff --git a/src/server/server.ts b/src/server/server.ts index 61738929..5709937f 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -218,17 +218,7 @@ export async function createServerContext( stream, email, _experimental_prompts: () => promptAPI.prompts, - prompts: { - getPrompt: (slug: T) => { - const prompt = promptAPI.prompts[slug]; - if (!prompt) { - throw new Error(`Prompt '${slug}' not found`); - } - return prompt; - }, - // Use the generated compile function with proper type safety - compile: promptAPI.prompts.compile, - }, + prompts: promptAPI.prompts, discord, objectstore, patchportal, diff --git a/src/types.ts b/src/types.ts index 5039c635..2154d4bf 100644 --- a/src/types.ts +++ b/src/types.ts @@ -933,15 +933,13 @@ export interface AgentContext { /** * prompts API for accessing and compiling prompts */ - prompts: { + prompts: import('./apis/prompt/generated/index.js').GeneratedPromptsCollection & { getPrompt< T extends keyof import('./apis/prompt/generated/index.js').GeneratedPromptsCollection, >( slug: T ): import('./apis/prompt/generated/index.js').GeneratedPromptsCollection[T]; - // Use the generated compile function with proper type safety - compile: import('./apis/prompt/generated/index.js').GeneratedPromptsCollection['compile']; }; } From 5995f8a2cfa4a75a111b38548defc8b02cf79da9 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Fri, 3 Oct 2025 14:55:47 -0400 Subject: [PATCH 04/26] added route 2 --- ...{build-and-copy.sh => build-clean-copy.sh} | 13 ++++-- src/types.ts | 40 ++++++++++++------- 2 files changed, 36 insertions(+), 17 deletions(-) rename scripts/{build-and-copy.sh => build-clean-copy.sh} (73%) diff --git a/scripts/build-and-copy.sh b/scripts/build-clean-copy.sh similarity index 73% rename from scripts/build-and-copy.sh rename to scripts/build-clean-copy.sh index 088c1900..8dfcd37d 100755 --- a/scripts/build-and-copy.sh +++ b/scripts/build-clean-copy.sh @@ -27,13 +27,20 @@ fi echo "โœ… Build completed successfully" +# Clean target project's dist directory +echo "๐Ÿงน Cleaning target project's dist directory..." +if [ -d "$TARGET_DIR" ]; then + rm -rf "$TARGET_DIR" + echo " Removed existing dist directory" +fi + # Create target directory if it doesn't exist echo "๐Ÿ“ Creating target directory: $TARGET_DIR" mkdir -p "$TARGET_DIR" -# Copy dist contents to target directory -echo "๐Ÿ“ฆ Copying dist/* to $TARGET_DIR" -cp -r dist/* "$TARGET_DIR/" +# Copy our newly built dist contents to target directory +echo "๐Ÿ“ฆ Copying newly built dist contents to $TARGET_DIR" +cp -r dist/. "$TARGET_DIR/" # Copy package.json and LICENSE.md echo "๐Ÿ“„ Copying package.json and LICENSE.md" diff --git a/src/types.ts b/src/types.ts index 2154d4bf..c21f7ad7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -811,6 +811,30 @@ export type WaitUntilCallback = ( promise: Promise | (() => void | Promise) ) => void; +/** + * Available prompt names + */ +export type PromptName = + keyof import('./apis/prompt/generated/index.js').GeneratedPromptsCollection; + +/** + * A prompt object with system and prompt functions + */ +export type PromptObject = { + system: import('./apis/prompt/generated/index.js').GeneratedPromptsCollection[T]['system']; + prompt: import('./apis/prompt/generated/index.js').GeneratedPromptsCollection[T]['prompt']; +}; + +/** + * The prompts API interface + */ +export interface PromptsAPI { + /** + * Get system or prompt functions by slug + */ + getPrompt: (name: T) => PromptObject; +} + export interface AgentContext { /** * the version of the Agentuity SDK @@ -926,21 +950,9 @@ export interface AgentContext { slack: SlackService; /** - * get the prompts collection for compiling dynamic prompts - */ - _experimental_prompts(): import('./apis/prompt/generated/index.js').GeneratedPromptsCollection; - - /** - * prompts API for accessing and compiling prompts + * EXPERIMENTAL: prompts API for accessing and compiling prompts */ - prompts: import('./apis/prompt/generated/index.js').GeneratedPromptsCollection & { - getPrompt< - T extends - keyof import('./apis/prompt/generated/index.js').GeneratedPromptsCollection, - >( - slug: T - ): import('./apis/prompt/generated/index.js').GeneratedPromptsCollection[T]; - }; + prompts: PromptsAPI; } /** From eb039d3d24ff5c86e2bee568282be7edd1ef9fa3 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Fri, 3 Oct 2025 15:04:30 -0400 Subject: [PATCH 05/26] remove unused file --- src/apis/prompt/generated/index.d.ts | 6 ---- src/apis/prompt/generated/index.js | 1 - src/apis/prompt/index.ts | 13 ------- src/apis/prompt/signature.ts | 51 ---------------------------- 4 files changed, 71 deletions(-) delete mode 100644 src/apis/prompt/signature.ts diff --git a/src/apis/prompt/generated/index.d.ts b/src/apis/prompt/generated/index.d.ts index 46e675d5..41f55581 100644 --- a/src/apis/prompt/generated/index.d.ts +++ b/src/apis/prompt/generated/index.d.ts @@ -8,11 +8,5 @@ export interface GeneratedPromptsCollection { [promptSlug: string]: any; } -// Placeholder signature collection that will be replaced -export type SignaturesCollection = { - [promptSlug: string]: (params: any) => string; -}; - // Placeholder exports that will be replaced export const prompts: GeneratedPromptsCollection = {} as any; -export const signatures: SignaturesCollection = {} as any; diff --git a/src/apis/prompt/generated/index.js b/src/apis/prompt/generated/index.js index 7e238212..9e641ac6 100644 --- a/src/apis/prompt/generated/index.js +++ b/src/apis/prompt/generated/index.js @@ -2,4 +2,3 @@ // Do not edit this file manually - it will be overwritten during the bundle process export const prompts = {}; -export const signatures = {}; diff --git a/src/apis/prompt/index.ts b/src/apis/prompt/index.ts index 3c9b4904..9e9aee55 100644 --- a/src/apis/prompt/index.ts +++ b/src/apis/prompt/index.ts @@ -5,10 +5,6 @@ import path from 'path'; import { pathToFileURL } from 'url'; import { PromptConfig, PromptName } from './generated/_index.js'; import type { PromptsCollection } from './generic_types.js'; -import { - createPromptSignatures, - type GetPromptSignatures, -} from './signature.js'; // Default empty prompts object const defaultPrompts = {}; @@ -20,12 +16,10 @@ interface GeneratedModule { export default class PromptAPI { public prompts: PromptsCollection; - public signatures: GetPromptSignatures; constructor() { // Initialize with empty prompts by default this.prompts = defaultPrompts; - this.signatures = createPromptSignatures(this.prompts); } /** @@ -134,8 +128,6 @@ export default class PromptAPI { // ); this.prompts = generatedModule.prompts || (defaultPrompts as PromptsCollection); - // Update signatures when prompts are loaded - this.signatures = createPromptSignatures(this.prompts); // console.log('Final prompts:', Object.keys(this.prompts)); } catch (error) { // Fallback to empty prompts if generated file doesn't exist @@ -144,8 +136,6 @@ export default class PromptAPI { error instanceof Error ? error.message : String(error) ); this.prompts = defaultPrompts; - // Update signatures with empty prompts - this.signatures = createPromptSignatures(this.prompts); console.warn( 'โš ๏ธ No generated prompts found. Run `agentuity bundle` to generate prompts from src/prompts.yaml' ); @@ -158,8 +148,5 @@ export { defaultPrompts, PromptConfig, PromptName }; export { GeneratedPromptsCollection, prompts, - SignaturesCollection, - signatures, } from './generated/index.js'; export * from './generic_types.js'; -export { createPromptSignature, createPromptSignatures } from './signature.js'; diff --git a/src/apis/prompt/signature.ts b/src/apis/prompt/signature.ts deleted file mode 100644 index a531cf77..00000000 --- a/src/apis/prompt/signature.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { Prompt, PromptSignature } from './generic_types.js'; - -/** - * Creates a signature function for a specific prompt type - * The function signature adapts based on whether the prompt has system, prompt, and variables - */ -export function createPromptSignature( - _prompt: T -): PromptSignature { - // This is a type-safe wrapper that will be replaced by the CLI with actual implementation - // The CLI will generate the appropriate function based on the prompt's structure - - // For now, return a generic function that handles all cases - // The CLI will replace this with the specific implementation - return ((..._args: unknown[]) => { - // This will be replaced by the CLI with the actual template compilation logic - throw new Error( - 'Generated signature function not found. Run `agentuity bundle` to generate prompts.' - ); - }) as unknown as PromptSignature; -} - -/** - * Type-safe prompt signature factory - * This function creates signature functions that match the exact structure of each prompt - */ -export function createPromptSignatures>( - prompts: T -): { - [K in keyof T]: PromptSignature; -} { - const signatures = {} as any; - - for (const [key, prompt] of Object.entries(prompts)) { - signatures[key] = createPromptSignature(prompt); - } - - return signatures; -} - -/** - * Utility type to extract the signature function type for a specific prompt - */ -export type GetPromptSignature = PromptSignature; - -/** - * Utility type to extract all signature functions from a prompts collection - */ -export type GetPromptSignatures> = { - [K in keyof T]: PromptSignature; -}; From 1629bbf08cb6e8f79cb064c9965ea63912c04fc2 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Fri, 3 Oct 2025 16:54:44 -0400 Subject: [PATCH 06/26] fix tests --- package.json | 2 +- src/apis/prompt/index.ts | 29 +++++++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index ff692d1b..2ec203d5 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ }, "scripts": { "build": "rm -rf dist && tsup && npm run types", - "build-and-copy": "./scripts/build-and-copy.sh", + "build-clean-copy": "./scripts/build-clean-copy.sh", "types": "tsc --emitDeclarationOnly --declaration", "bun:start": "npm run build && bun run ./dist/index.js", "node:start": "npm run build && node ./dist/index.js", diff --git a/src/apis/prompt/index.ts b/src/apis/prompt/index.ts index 9e9aee55..1497ae99 100644 --- a/src/apis/prompt/index.ts +++ b/src/apis/prompt/index.ts @@ -3,7 +3,6 @@ import fs from 'fs/promises'; import path from 'path'; import { pathToFileURL } from 'url'; -import { PromptConfig, PromptName } from './generated/_index.js'; import type { PromptsCollection } from './generic_types.js'; // Default empty prompts object @@ -144,9 +143,27 @@ export default class PromptAPI { } // Re-export generated types and prompts (following POC pattern) -export { defaultPrompts, PromptConfig, PromptName }; -export { - GeneratedPromptsCollection, - prompts, -} from './generated/index.js'; +export { defaultPrompts }; + +// Conditional exports for generated content +let PromptConfig: any; +let PromptName: any; +let GeneratedPromptsCollection: any; +let prompts: any; + +try { + const generatedModule = require('./generated/_index.js'); + PromptConfig = generatedModule.PromptConfig; + PromptName = generatedModule.PromptName; + GeneratedPromptsCollection = generatedModule.GeneratedPromptsCollection; + prompts = generatedModule.prompts; +} catch { + // Fallback to placeholder values when generated content doesn't exist + PromptConfig = {}; + PromptName = {}; + GeneratedPromptsCollection = {}; + prompts = {}; +} + +export { PromptConfig, PromptName, GeneratedPromptsCollection, prompts }; export * from './generic_types.js'; From ed0a53666873fcd846d2ae5423ccfa44497b80e4 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Fri, 3 Oct 2025 17:54:18 -0400 Subject: [PATCH 07/26] added fix --- src/server/server.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/server.ts b/src/server/server.ts index 5709937f..dbbadb6f 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -218,7 +218,9 @@ export async function createServerContext( stream, email, _experimental_prompts: () => promptAPI.prompts, - prompts: promptAPI.prompts, + prompts: { + getPrompt: (name: string) => promptAPI.prompts[name], + }, discord, objectstore, patchportal, From db0db36b31aa23e9cc66fe7a3b37c1ed6901b166 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Fri, 3 Oct 2025 18:02:27 -0400 Subject: [PATCH 08/26] added version --- CHANGELOG.md | 6 ++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 208d63f9..87dc9db4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # @agentuity/sdk Changelog +## 0.0.149 + +### Patch Changes + +- Changed up the function signature and add a couple features + ## [0.0.148] ### Patch Changes diff --git a/package-lock.json b/package-lock.json index d040ab0a..25ea6ae5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@agentuity/sdk", - "version": "0.0.148", + "version": "0.0.149", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@agentuity/sdk", - "version": "0.0.148", + "version": "0.0.149", "license": "Apache-2.0", "dependencies": { "@opentelemetry/api": "^1.9.0", diff --git a/package.json b/package.json index 2ec203d5..c486f302 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agentuity/sdk", - "version": "0.0.148", + "version": "0.0.149", "description": "The Agentuity SDK for NodeJS and Bun", "license": "Apache-2.0", "public": true, From 966137fb37dfd765f9b9b8d535e691a722221035 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Fri, 3 Oct 2025 19:32:08 -0400 Subject: [PATCH 09/26] added patching logic --- src/apis/patchportal.ts | 101 +++++++++++++++++++++++++++++++++--- src/index.ts | 5 ++ src/utils/promptMetadata.ts | 71 +++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 src/utils/promptMetadata.ts diff --git a/src/apis/patchportal.ts b/src/apis/patchportal.ts index bfa8c835..21d85cc6 100644 --- a/src/apis/patchportal.ts +++ b/src/apis/patchportal.ts @@ -1,36 +1,123 @@ +// Global instance storage to ensure true singleton across all module contexts +declare global { + var __patchPortalInstance: PatchPortal | undefined; +} + /** * Singleton class for PatchPortal */ export default class PatchPortal { - private static instance: PatchPortal | null = null; private state: Record = {}; + private instanceId: string; private constructor() { // Private constructor to prevent direct instantiation + this.instanceId = `PatchPortal-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + console.log( + '๐Ÿ—๏ธ PatchPortal constructor called, instanceId:', + this.instanceId + ); } /** * Get the singleton instance of PatchPortal */ public static async getInstance(): Promise { - if (!PatchPortal.instance) { - PatchPortal.instance = new PatchPortal(); + console.debug('๐Ÿ” PatchPortal.getInstance() called'); + console.debug( + '๐Ÿ” globalThis.__patchPortalInstance exists:', + !!globalThis.__patchPortalInstance + ); + + if (!globalThis.__patchPortalInstance) { + globalThis.__patchPortalInstance = new PatchPortal(); + console.debug( + '๐Ÿ†• Created new PatchPortal instance, ID:', + globalThis.__patchPortalInstance.instanceId + ); + console.debug( + '๐Ÿ” Global state after creation:', + Object.keys(globalThis).filter((k) => k.includes('patch')) + ); + } else { + console.debug( + 'โ™ป๏ธ Returning existing PatchPortal instance, ID:', + globalThis.__patchPortalInstance.instanceId + ); + console.debug( + '๐Ÿ” Current state keys:', + Object.keys(globalThis.__patchPortalInstance.state) + ); } - return PatchPortal.instance; + return globalThis.__patchPortalInstance; } /** * Example method - you can add your specific functionality here */ - public async process(key: string, data: unknown): Promise { + public async set(key: string, data: T): Promise { + console.debug('๐Ÿ” PatchPortal.set() called with key:', key); + console.debug('๐Ÿ” Instance ID:', this.instanceId); + console.debug('๐Ÿ” State before set:', Object.keys(this.state)); this.state[key] = data; - return data; + console.debug('๐Ÿ” State after set:', Object.keys(this.state)); + console.debug( + '๐Ÿ” Data stored:', + typeof data === 'object' ? Object.keys(data as any) : typeof data + ); + } + + public async get(key: string): Promise { + console.debug('๐Ÿ” PatchPortal.get() called with key:', key); + console.debug('๐Ÿ” Instance ID:', this.instanceId); + console.debug('๐Ÿ” Current state keys:', Object.keys(this.state)); + console.debug('๐Ÿ” Key exists:', key in this.state); + const result = this.state[key] as T; + console.debug( + '๐Ÿ” Retrieved data:', + result + ? typeof result === 'object' + ? Object.keys(result as any) + : typeof result + : 'undefined' + ); + return result; + } + + /** + * Print out the whole state of the PatchPortal + */ + public printState(): void { + console.debug('๐Ÿ” PatchPortal.printState() called'); + console.debug('๐Ÿ” Instance ID:', this.instanceId); + console.log('๐Ÿ” PatchPortal State:'); + console.log('๐Ÿ“Š Total keys:', Object.keys(this.state).length); + console.log('๐Ÿ“‹ All keys:', Object.keys(this.state)); + console.log('๐Ÿ“ฆ Full state:', JSON.stringify(this.state, null, 2)); + console.debug( + '๐Ÿ” Global instance check:', + globalThis.__patchPortalInstance === this + ); + } + + /** + * Get all keys in the PatchPortal + */ + public getAllKeys(): string[] { + return Object.keys(this.state); + } + + /** + * Get the entire state object + */ + public getState(): Record { + return { ...this.state }; } /** * Example method for demonstrating the singleton */ public getInstanceId(): string { - return `PatchPortal-${Date.now()}`; + return this.instanceId; } } diff --git a/src/index.ts b/src/index.ts index ecc5ef08..11a26aba 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,11 @@ export * from './logger'; export * from './server'; export * from './types'; export * from './utils/interpolate'; +export { + type PromptAttributes, + type PromptAttributesParams, + processPromptMetadata, +} from './utils/promptMetadata'; import DiscordAPI from './apis/discord'; // Export APIs diff --git a/src/utils/promptMetadata.ts b/src/utils/promptMetadata.ts new file mode 100644 index 00000000..d52b4be7 --- /dev/null +++ b/src/utils/promptMetadata.ts @@ -0,0 +1,71 @@ +import crypto from 'crypto'; +import PatchPortal from '../apis/patchportal.js'; + +export interface PromptAttributesParams { + slug: string; + compiled: string; + template: string; + variables?: Record; +} + +export interface PromptAttributes extends PromptAttributesParams { + hash: string; + compiledHash: string; +} + +/** + * Process a prompt and store its metadata in PatchPortal + */ +export async function processPromptMetadata( + attributes: PromptAttributesParams +): Promise { + console.log('๐Ÿ”ง processPromptMetadata called with:', { + slug: attributes.slug, + template: attributes.template?.substring(0, 50) + '...', + compiled: attributes.compiled?.substring(0, 50) + '...', + variables: attributes.variables, + }); + + const patchPortal = await PatchPortal.getInstance(); + console.log('โœ… PatchPortal instance obtained'); + + // Generate hash + const hash = crypto + .createHash('sha256') + .update(attributes.template) + .digest('hex'); + + console.log('๐Ÿ”‘ Template hash:', hash); + + const compiledHash = crypto + .createHash('sha256') + .update(attributes.compiled) + .digest('hex'); + + console.log('๐Ÿ”‘ Compiled hash:', compiledHash); + + // Create metadata object + const metadata = { + ...attributes, + hash, + compiledHash, + }; + + console.log('๐Ÿ“ฆ Created metadata object:', { + slug: metadata.slug, + hash: metadata.hash, + compiledHash: metadata.compiledHash, + timestamp: new Date().toISOString(), + }); + + // Store in PatchPortal using compiled hash as key + const key = `prompt:${compiledHash}`; + console.log('๐Ÿ”‘ Storing with key:', key); + + await patchPortal.set(key, metadata); + console.log('โœ… Metadata stored successfully in PatchPortal'); + + // Print state after storing + console.log('๐Ÿ“Š PatchPortal state after storing:'); + patchPortal.printState(); +} From 5844b067854ea2e07494cbb91e70aff448c9af19 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Sat, 4 Oct 2025 13:22:34 -0400 Subject: [PATCH 10/26] added a new logger --- src/apis/patchportal.ts | 50 +++++----- src/apis/prompt/index.ts | 25 ++--- src/logger/index.ts | 1 + src/logger/internal.ts | 194 ++++++++++++++++++++++++++++++++++++ src/logger/user.ts | 11 ++ src/router/router.ts | 5 +- src/server/server.ts | 3 +- src/utils/promptMetadata.ts | 17 ++-- 8 files changed, 259 insertions(+), 47 deletions(-) create mode 100644 src/logger/internal.ts create mode 100644 src/logger/user.ts diff --git a/src/apis/patchportal.ts b/src/apis/patchportal.ts index 21d85cc6..501c875d 100644 --- a/src/apis/patchportal.ts +++ b/src/apis/patchportal.ts @@ -1,3 +1,5 @@ +import { internal } from '../logger/internal'; + // Global instance storage to ensure true singleton across all module contexts declare global { var __patchPortalInstance: PatchPortal | undefined; @@ -13,7 +15,7 @@ export default class PatchPortal { private constructor() { // Private constructor to prevent direct instantiation this.instanceId = `PatchPortal-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; - console.log( + internal.debug( '๐Ÿ—๏ธ PatchPortal constructor called, instanceId:', this.instanceId ); @@ -23,28 +25,28 @@ export default class PatchPortal { * Get the singleton instance of PatchPortal */ public static async getInstance(): Promise { - console.debug('๐Ÿ” PatchPortal.getInstance() called'); - console.debug( + internal.debug('๐Ÿ” PatchPortal.getInstance() called'); + internal.debug( '๐Ÿ” globalThis.__patchPortalInstance exists:', !!globalThis.__patchPortalInstance ); if (!globalThis.__patchPortalInstance) { globalThis.__patchPortalInstance = new PatchPortal(); - console.debug( + internal.debug( '๐Ÿ†• Created new PatchPortal instance, ID:', globalThis.__patchPortalInstance.instanceId ); - console.debug( + internal.debug( '๐Ÿ” Global state after creation:', Object.keys(globalThis).filter((k) => k.includes('patch')) ); } else { - console.debug( + internal.debug( 'โ™ป๏ธ Returning existing PatchPortal instance, ID:', globalThis.__patchPortalInstance.instanceId ); - console.debug( + internal.debug( '๐Ÿ” Current state keys:', Object.keys(globalThis.__patchPortalInstance.state) ); @@ -56,24 +58,24 @@ export default class PatchPortal { * Example method - you can add your specific functionality here */ public async set(key: string, data: T): Promise { - console.debug('๐Ÿ” PatchPortal.set() called with key:', key); - console.debug('๐Ÿ” Instance ID:', this.instanceId); - console.debug('๐Ÿ” State before set:', Object.keys(this.state)); + internal.debug('๐Ÿ” PatchPortal.set() called with key:', key); + internal.debug('๐Ÿ” Instance ID:', this.instanceId); + internal.debug('๐Ÿ” State before set:', Object.keys(this.state)); this.state[key] = data; - console.debug('๐Ÿ” State after set:', Object.keys(this.state)); - console.debug( + internal.debug('๐Ÿ” State after set:', Object.keys(this.state)); + internal.debug( '๐Ÿ” Data stored:', typeof data === 'object' ? Object.keys(data as any) : typeof data ); } public async get(key: string): Promise { - console.debug('๐Ÿ” PatchPortal.get() called with key:', key); - console.debug('๐Ÿ” Instance ID:', this.instanceId); - console.debug('๐Ÿ” Current state keys:', Object.keys(this.state)); - console.debug('๐Ÿ” Key exists:', key in this.state); + internal.debug('๐Ÿ” PatchPortal.get() called with key:', key); + internal.debug('๐Ÿ” Instance ID:', this.instanceId); + internal.debug('๐Ÿ” Current state keys:', Object.keys(this.state)); + internal.debug('๐Ÿ” Key exists:', key in this.state); const result = this.state[key] as T; - console.debug( + internal.debug( '๐Ÿ” Retrieved data:', result ? typeof result === 'object' @@ -88,13 +90,13 @@ export default class PatchPortal { * Print out the whole state of the PatchPortal */ public printState(): void { - console.debug('๐Ÿ” PatchPortal.printState() called'); - console.debug('๐Ÿ” Instance ID:', this.instanceId); - console.log('๐Ÿ” PatchPortal State:'); - console.log('๐Ÿ“Š Total keys:', Object.keys(this.state).length); - console.log('๐Ÿ“‹ All keys:', Object.keys(this.state)); - console.log('๐Ÿ“ฆ Full state:', JSON.stringify(this.state, null, 2)); - console.debug( + internal.debug('๐Ÿ” PatchPortal.printState() called'); + internal.debug('๐Ÿ” Instance ID:', this.instanceId); + internal.debug('๐Ÿ” PatchPortal State:'); + internal.debug('๐Ÿ“Š Total keys:', Object.keys(this.state).length); + internal.debug('๐Ÿ“‹ All keys:', Object.keys(this.state)); + internal.debug('๐Ÿ“ฆ Full state:', JSON.stringify(this.state, null, 2)); + internal.debug( '๐Ÿ” Global instance check:', globalThis.__patchPortalInstance === this ); diff --git a/src/apis/prompt/index.ts b/src/apis/prompt/index.ts index 1497ae99..12956886 100644 --- a/src/apis/prompt/index.ts +++ b/src/apis/prompt/index.ts @@ -3,6 +3,7 @@ import fs from 'fs/promises'; import path from 'path'; import { pathToFileURL } from 'url'; +import { internal } from '../../logger/internal'; import type { PromptsCollection } from './generic_types.js'; // Default empty prompts object @@ -84,7 +85,7 @@ export default class PromptAPI { // Method to load prompts dynamically (called by context) public async loadPrompts(): Promise { - // console.log('loadPrompts() called'); + internal.debug('loadPrompts() called'); try { // Try multiple possible paths for the generated prompts let generatedModule: unknown; @@ -92,9 +93,9 @@ export default class PromptAPI { // Dynamic module resolution strategy const possiblePaths = await this.resolveGeneratedPaths(); - // console.log('Trying absolute paths:'); + internal.debug('Trying absolute paths:', possiblePaths); for (const possiblePath of possiblePaths) { - // console.log(' Checking:', possiblePath); + internal.debug(' Checking:', possiblePath); try { await fs.access(possiblePath); // Get file stats for cache-busting @@ -106,7 +107,7 @@ export default class PromptAPI { // Use ESM dynamic import instead of require generatedModule = await import(fileUrl); - // console.log(' Successfully loaded from:', possiblePath); + internal.debug(' Successfully loaded from:', possiblePath); break; } catch {} } @@ -120,22 +121,22 @@ export default class PromptAPI { throw new Error('Generated module has invalid shape'); } - // console.log('Generated module:', generatedModule); - // console.log( - // 'Prompts in module:', - // Object.keys(generatedModule.prompts || {}) - // ); + internal.debug('Generated module:', generatedModule); + internal.debug( + 'Prompts in module:', + Object.keys(generatedModule.prompts || {}) + ); this.prompts = generatedModule.prompts || (defaultPrompts as PromptsCollection); - // console.log('Final prompts:', Object.keys(this.prompts)); + internal.debug('Final prompts:', Object.keys(this.prompts)); } catch (error) { // Fallback to empty prompts if generated file doesn't exist - console.log( + internal.error( 'Error loading prompts:', error instanceof Error ? error.message : String(error) ); this.prompts = defaultPrompts; - console.warn( + internal.warn( 'โš ๏ธ No generated prompts found. Run `agentuity bundle` to generate prompts from src/prompts.yaml' ); } diff --git a/src/logger/index.ts b/src/logger/index.ts index 1ff09efd..c03033b6 100644 --- a/src/logger/index.ts +++ b/src/logger/index.ts @@ -1 +1,2 @@ export * from './logger'; +export { type Logger, logger } from './user'; diff --git a/src/logger/internal.ts b/src/logger/internal.ts new file mode 100644 index 00000000..6fc086f8 --- /dev/null +++ b/src/logger/internal.ts @@ -0,0 +1,194 @@ +import type { Json } from '../types'; + +/** + * Log levels for internal SDK logging + */ +export type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'silent'; + +/** + * Internal logger configuration + */ +interface InternalLoggerConfig { + level: LogLevel; + context?: Record; +} + +/** + * Simple internal logger that doesn't depend on other SDK modules + * This logger is only for SDK internal diagnostics and debugging + */ +class InternalLogger { + private config: InternalLoggerConfig; + + constructor() { + this.config = this.loadConfig(); + } + + /** + * Load configuration from environment variables + */ + private loadConfig(): InternalLoggerConfig { + const envLevel = process.env.AGENTUITY_SDK_LOG_LEVEL?.toLowerCase(); + + // Validate log level + const validLevels: LogLevel[] = [ + 'debug', + 'info', + 'warn', + 'error', + 'silent', + ]; + const level = validLevels.includes(envLevel as LogLevel) + ? (envLevel as LogLevel) + : 'silent'; + + return { + level, + context: { + '@agentuity/source': 'sdk-internal', + '@agentuity/timestamp': new Date().toISOString(), + }, + }; + } + + /** + * Check if a log level should be output based on current configuration + */ + private shouldLog(level: LogLevel): boolean { + if (this.config.level === 'silent') return false; + + const levelPriority = { + debug: 0, + info: 1, + warn: 2, + error: 3, + silent: 4, + }; + + return levelPriority[level] >= levelPriority[this.config.level]; + } + + /** + * Format a log message with context + */ + private formatMessage(message: unknown, ...args: unknown[]): string { + const contextStr = + this.config.context && Object.keys(this.config.context).length > 0 + ? Object.entries(this.config.context) + .map( + ([key, value]) => + `${key}=${typeof value === 'object' ? JSON.stringify(value) : value}` + ) + .join(' ') + : ''; + + const formattedMessage = + typeof message === 'string' ? message : JSON.stringify(message); + const argsStr = + args.length > 0 + ? ' ' + + args + .map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))) + .join(' ') + : ''; + + return `[INTERNAL] ${formattedMessage}${argsStr}${contextStr ? ` [${contextStr}]` : ''}`; + } + + /** + * Log a debug message + */ + debug(message: unknown, ...args: unknown[]): void { + if (this.shouldLog('debug')) { + console.log(this.formatMessage(message, ...args)); + } + } + + /** + * Log an info message + */ + info(message: unknown, ...args: unknown[]): void { + if (this.shouldLog('info')) { + console.log(this.formatMessage(message, ...args)); + } + } + + /** + * Log a warning message + */ + warn(message: unknown, ...args: unknown[]): void { + if (this.shouldLog('warn')) { + console.log(this.formatMessage(message, ...args)); + } + } + + /** + * Log an error message + */ + error(message: unknown, ...args: unknown[]): void { + if (this.shouldLog('error')) { + console.log(this.formatMessage(message, ...args)); + } + } + + /** + * Update configuration at runtime + */ + updateConfig(config: Partial): void { + this.config = { ...this.config, ...config }; + } + + /** + * Get current configuration + */ + getConfig(): InternalLoggerConfig { + return { ...this.config }; + } + + /** + * Check if logging is enabled + */ + isEnabled(): boolean { + return this.config.level !== 'silent'; + } + + /** + * Create a child logger with additional context + */ + child(context: Record): InternalLogger { + const childLogger = new InternalLogger(); + childLogger.updateConfig({ + ...this.config, + context: { + ...this.config.context, + ...context, + }, + }); + return childLogger; + } +} + +// Singleton instance - not exported +const internalLogger = new InternalLogger(); + +/** + * Internal logger for SDK use only + * This is NOT exported from the main SDK index + */ +export const internal = { + debug: (message: unknown, ...args: unknown[]) => + internalLogger.debug(message, ...args), + info: (message: unknown, ...args: unknown[]) => + internalLogger.info(message, ...args), + warn: (message: unknown, ...args: unknown[]) => + internalLogger.warn(message, ...args), + error: (message: unknown, ...args: unknown[]) => + internalLogger.error(message, ...args), + + // Utility methods + updateConfig: (config: Partial) => + internalLogger.updateConfig(config), + getConfig: () => internalLogger.getConfig(), + isEnabled: () => internalLogger.isEnabled(), + child: (context: Record) => internalLogger.child(context), +}; diff --git a/src/logger/user.ts b/src/logger/user.ts new file mode 100644 index 00000000..c75097b9 --- /dev/null +++ b/src/logger/user.ts @@ -0,0 +1,11 @@ +import ConsoleLogger from './console'; +import type { Logger } from './logger'; + +/** + * User-facing logger instance + * This is the logger that SDK consumers should use + */ +export const logger: Logger = new ConsoleLogger(); + +// Re-export the Logger type for convenience +export type { Logger } from './logger'; diff --git a/src/router/router.ts b/src/router/router.ts index 34d636ad..abd9109d 100644 --- a/src/router/router.ts +++ b/src/router/router.ts @@ -11,6 +11,7 @@ import { ValueType, } from '@opentelemetry/api'; import type { Logger } from '../logger'; +import { internal } from '../logger/internal'; import AgentResolver from '../server/agents'; import { HandlerParameterProvider } from '../server/handlerParameterProvider'; import type { ServerRequest, ServerRoute } from '../server/types'; @@ -25,9 +26,9 @@ import type { RemoteAgent, Stream, } from '../types'; +import AgentContextWaitUntilHandler from './context'; import AgentRequestHandler from './request'; import AgentResponseHandler from './response'; -import AgentContextWaitUntilHandler from './context'; interface RouterConfig { handler: AgentHandler; @@ -115,7 +116,7 @@ export function recordException(span: Span, ex: unknown, skipLog = false) { if (store?.logger) { store.logger.error('%s', ex); } else { - console.error(ex); + internal.error(ex); } } __exception.__exception_recorded = true; diff --git a/src/server/server.ts b/src/server/server.ts index dbbadb6f..93e41a62 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -10,6 +10,7 @@ import PromptAPI from '../apis/prompt/index.js'; import StreamAPIImpl from '../apis/stream'; import VectorAPI from '../apis/vector'; import type { Logger } from '../logger'; +import { internal } from '../logger/internal'; import { createRouter } from '../router'; import type { AgentConfig, @@ -58,7 +59,7 @@ async function createRoute( try { mod = await import(filename); } catch (error) { - console.error('Error importing module', error); + internal.error('Error importing module', error); throw new Error(`Error importing module ${filename}: ${error}`); } diff --git a/src/utils/promptMetadata.ts b/src/utils/promptMetadata.ts index d52b4be7..3d5ecd9c 100644 --- a/src/utils/promptMetadata.ts +++ b/src/utils/promptMetadata.ts @@ -1,5 +1,6 @@ import crypto from 'crypto'; import PatchPortal from '../apis/patchportal.js'; +import { internal } from '../logger/internal'; export interface PromptAttributesParams { slug: string; @@ -19,7 +20,7 @@ export interface PromptAttributes extends PromptAttributesParams { export async function processPromptMetadata( attributes: PromptAttributesParams ): Promise { - console.log('๐Ÿ”ง processPromptMetadata called with:', { + internal.debug('๐Ÿ”ง processPromptMetadata called with:', { slug: attributes.slug, template: attributes.template?.substring(0, 50) + '...', compiled: attributes.compiled?.substring(0, 50) + '...', @@ -27,7 +28,7 @@ export async function processPromptMetadata( }); const patchPortal = await PatchPortal.getInstance(); - console.log('โœ… PatchPortal instance obtained'); + internal.debug('โœ… PatchPortal instance obtained'); // Generate hash const hash = crypto @@ -35,14 +36,14 @@ export async function processPromptMetadata( .update(attributes.template) .digest('hex'); - console.log('๐Ÿ”‘ Template hash:', hash); + internal.debug('๐Ÿ”‘ Template hash:', hash); const compiledHash = crypto .createHash('sha256') .update(attributes.compiled) .digest('hex'); - console.log('๐Ÿ”‘ Compiled hash:', compiledHash); + internal.debug('๐Ÿ”‘ Compiled hash:', compiledHash); // Create metadata object const metadata = { @@ -51,7 +52,7 @@ export async function processPromptMetadata( compiledHash, }; - console.log('๐Ÿ“ฆ Created metadata object:', { + internal.debug('๐Ÿ“ฆ Created metadata object:', { slug: metadata.slug, hash: metadata.hash, compiledHash: metadata.compiledHash, @@ -60,12 +61,12 @@ export async function processPromptMetadata( // Store in PatchPortal using compiled hash as key const key = `prompt:${compiledHash}`; - console.log('๐Ÿ”‘ Storing with key:', key); + internal.debug('๐Ÿ”‘ Storing with key:', key); await patchPortal.set(key, metadata); - console.log('โœ… Metadata stored successfully in PatchPortal'); + internal.debug('โœ… Metadata stored successfully in PatchPortal'); // Print state after storing - console.log('๐Ÿ“Š PatchPortal state after storing:'); + internal.debug('๐Ÿ“Š PatchPortal state after storing:'); patchPortal.printState(); } From a854142a01ff624df5035a6abab6c2c4c8feb273 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Mon, 6 Oct 2025 09:59:00 -0400 Subject: [PATCH 11/26] add eval execution --- package-lock.json | 19 ++++ package.json | 1 + src/apis/eval.ts | 215 ++++++++++++++++++++++++++++++++++++ src/index.ts | 13 ++- src/utils/promptMetadata.ts | 3 +- 5 files changed, 249 insertions(+), 2 deletions(-) create mode 100644 src/apis/eval.ts diff --git a/package-lock.json b/package-lock.json index 25ea6ae5..4324e57a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.149", "license": "Apache-2.0", "dependencies": { + "@clickhouse/client": "^1.0.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/api-logs": "^0.57.2", "@opentelemetry/auto-instrumentations-node": "^0.56.1", @@ -1108,6 +1109,24 @@ "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", "license": "MIT" }, + "node_modules/@clickhouse/client": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@clickhouse/client/-/client-1.12.1.tgz", + "integrity": "sha512-7ORY85rphRazqHzImNXMrh4vsaPrpetFoTWpZYueCO2bbO6PXYDXp/GQ4DgxnGIqbWB/Di1Ai+Xuwq2o7DJ36A==", + "license": "Apache-2.0", + "dependencies": { + "@clickhouse/client-common": "1.12.1" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@clickhouse/client-common": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@clickhouse/client-common/-/client-common-1.12.1.tgz", + "integrity": "sha512-ccw1N6hB4+MyaAHIaWBwGZ6O2GgMlO99FlMj0B0UEGfjxM9v5dYVYql6FpP19rMwrVAroYs/IgX2vyZEBvzQLg==", + "license": "Apache-2.0" + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.25.9", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.9.tgz", diff --git a/package.json b/package.json index c486f302..5554104b 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "esbuild": ">=0.25.0" }, "dependencies": { + "@clickhouse/client": "^1.0.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/api-logs": "^0.57.2", "@opentelemetry/auto-instrumentations-node": "^0.56.1", diff --git a/src/apis/eval.ts b/src/apis/eval.ts new file mode 100644 index 00000000..0666bf15 --- /dev/null +++ b/src/apis/eval.ts @@ -0,0 +1,215 @@ +import { createClient } from '@clickhouse/client'; +import { internal } from '../logger/internal'; + +// Eval SDK types +export interface EvalRequest { + input: string; + output: string; + spanId: string; +} + +export interface EvalResponse { + pass: ( + value: boolean, + metadata?: { reasoning?: string; [key: string]: unknown } + ) => void; + score: ( + value: number, + metadata?: { reasoning?: string; [key: string]: unknown } + ) => void; // 0 to 1 +} + +export interface EvalContext { + // optional for now + [key: string]: unknown; +} + +export type EvalFunction = ( + ctx: EvalContext, + req: EvalRequest, + res: EvalResponse +) => Promise; + +// Eval result types +export interface EvalResult { + spanId: string; + resultType: 'pass' | 'fail' | 'score'; + scoreValue?: number; + metadata?: { reasoning?: string; [key: string]: unknown }; + timestamp: Date; +} + +export interface EvalRunnerConfig { + clickhouseHost: string; + clickhouseUser: string; + clickhousePassword: string; + spansTable?: string; + resultsTable?: string; +} + +export default class EvalAPI { + private config: EvalRunnerConfig; + private client: ReturnType; + + constructor(config: EvalRunnerConfig) { + this.config = { + spansTable: 'spans', + resultsTable: 'eval_results', + ...config, + }; + + this.client = createClient({ + host: this.config.clickhouseHost, + username: this.config.clickhouseUser, + password: this.config.clickhousePassword, + }); + } + + /** + * Fetch span data from ClickHouse and transform it for eval + */ + async fetchSpan(spanId: string): Promise { + internal.debug(`Fetching span ${spanId} from ClickHouse`); + + const query = ` + SELECT input, output + FROM ${this.config.spansTable} + WHERE spanId = {spanId:String} + LIMIT 1 + `; + + const result = await this.client.query({ + query, + query_params: { spanId }, + }); + + const response = await result.json<{ input: string; output: string }>(); + const row = response.data?.[0]; + + if (!row) { + throw new Error(`No span found for id ${spanId}`); + } + + return { + input: row.input, + output: row.output, + spanId, + }; + } + + /** + * Write eval result back to ClickHouse + */ + async writeResult(result: EvalResult): Promise { + internal.debug(`Writing eval result for span ${result.spanId}:`, result); + + await this.client.insert({ + table: this.config.resultsTable || 'eval_results', + values: [ + { + spanId: result.spanId, + resultType: result.resultType, + scoreValue: result.scoreValue ?? null, + metadata: result.metadata ? JSON.stringify(result.metadata) : null, + timestamp: result.timestamp.toISOString(), + }, + ], + format: 'JSONEachRow', + }); + } + + /** + * Load eval function from file path + */ + async loadEval(evalPath: string): Promise { + internal.debug(`Loading eval function from ${evalPath}`); + + try { + const module = await import(evalPath); + return module.default; + } catch (error) { + throw new Error( + `Failed to load eval function from ${evalPath}: ${error}` + ); + } + } + + /** + * Run eval on a span + */ + async runEval(spanId: string, evalPath: string): Promise { + internal.debug(`Running eval for span ${spanId} with function ${evalPath}`); + + // Load span data + const request = await this.fetchSpan(spanId); + + // Prepare response object + let resultType: 'pass' | 'fail' | 'score' = 'fail'; + let scoreValue: number | undefined; + let metadata: { reasoning?: string; [key: string]: unknown } | undefined; + + const response: EvalResponse = { + pass: ( + value: boolean, + meta?: { reasoning?: string; [key: string]: unknown } + ) => { + resultType = value ? 'pass' : 'fail'; + metadata = meta; + }, + score: ( + val: number, + meta?: { reasoning?: string; [key: string]: unknown } + ) => { + resultType = 'score'; + scoreValue = val; + metadata = meta; + }, + }; + + const context: EvalContext = {}; + + // Load and run eval function + const evaluate = await this.loadEval(evalPath); + await evaluate(context, request, response); + + // Create result + const result: EvalResult = { + spanId, + resultType, + scoreValue, + metadata, + timestamp: new Date(), + }; + + // Write result to database + await this.writeResult(result); + + internal.debug( + `Eval complete for span ${spanId} -> ${resultType}${scoreValue !== undefined ? ` (${scoreValue})` : ''}` + ); + + return result; + } + + /** + * Create the eval results table if it doesn't exist + */ + async createResultsTable(): Promise { + const createTableQuery = ` + CREATE TABLE IF NOT EXISTS ${this.config.resultsTable} ( + spanId String, + resultType String, + scoreValue Float64, + metadata String, + timestamp DateTime DEFAULT now() + ) ENGINE = MergeTree() + ORDER BY (spanId, timestamp) + `; + + await this.client.command({ + query: createTableQuery, + }); + + internal.debug(`Created eval results table: ${this.config.resultsTable}`); + } +} diff --git a/src/index.ts b/src/index.ts index 11a26aba..aabb9a79 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,13 +8,24 @@ export { processPromptMetadata, } from './utils/promptMetadata'; +// Export eval types +export type { + EvalRequest, + EvalResponse, + EvalContext, + EvalFunction, + EvalResult, + EvalRunnerConfig, +} from './apis/eval'; + import DiscordAPI from './apis/discord'; // Export APIs import EmailAPI from './apis/email'; +import EvalAPI from './apis/eval'; import PatchPortal from './apis/patchportal'; import PromptAPI from './apis/prompt'; import StreamAPIImpl from './apis/stream'; -export { EmailAPI, DiscordAPI, PatchPortal, PromptAPI, StreamAPIImpl }; +export { EmailAPI, DiscordAPI, EvalAPI, PatchPortal, PromptAPI, StreamAPIImpl }; import { TeamsActivityHandler } from 'botbuilder'; import { run } from './autostart'; diff --git a/src/utils/promptMetadata.ts b/src/utils/promptMetadata.ts index 3d5ecd9c..3e598f56 100644 --- a/src/utils/promptMetadata.ts +++ b/src/utils/promptMetadata.ts @@ -3,6 +3,7 @@ import PatchPortal from '../apis/patchportal.js'; import { internal } from '../logger/internal'; export interface PromptAttributesParams { + promptId?: string; slug: string; compiled: string; template: string; @@ -10,7 +11,7 @@ export interface PromptAttributesParams { } export interface PromptAttributes extends PromptAttributesParams { - hash: string; + tempalteHash: string; compiledHash: string; } From d7b40665ee32d82dc8cc44d17503b382c43ba066 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Mon, 13 Oct 2025 10:57:56 -0400 Subject: [PATCH 12/26] aded eval in metadata --- src/utils/promptMetadata.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/utils/promptMetadata.ts b/src/utils/promptMetadata.ts index 9f3b58dd..22e8dffc 100644 --- a/src/utils/promptMetadata.ts +++ b/src/utils/promptMetadata.ts @@ -7,6 +7,7 @@ export interface PromptAttributesParams { compiled: string; template: string; variables?: Record; + evals?: string[]; } export interface PromptAttributes extends PromptAttributesParams { @@ -26,6 +27,7 @@ export async function processPromptMetadata( template: attributes.template?.substring(0, 50) + '...', compiled: attributes.compiled?.substring(0, 50) + '...', variables: attributes.variables, + evals: attributes.evals, }); const patchPortal = await PatchPortal.getInstance(); @@ -57,6 +59,7 @@ export async function processPromptMetadata( slug: metadata.slug, templateHash: metadata.templateHash, compiledHash: metadata.compiledHash, + evals: metadata.evals, timestamp: new Date().toISOString(), }); From d74c18059272d760f9537f9240904b24495a6f4e Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Tue, 14 Oct 2025 10:35:40 -0400 Subject: [PATCH 13/26] added some changes --- src/apis/eval.ts | 155 +++++++++++++-------------------------------- src/index.ts | 9 +++ src/server/bun.ts | 41 +++++++++++- src/server/node.ts | 39 +++++++++++- 4 files changed, 130 insertions(+), 114 deletions(-) diff --git a/src/apis/eval.ts b/src/apis/eval.ts index 0666bf15..7b23bd2a 100644 --- a/src/apis/eval.ts +++ b/src/apis/eval.ts @@ -1,11 +1,13 @@ -import { createClient } from '@clickhouse/client'; +import fs from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; import { internal } from '../logger/internal'; // Eval SDK types export interface EvalRequest { input: string; output: string; - spanId: string; + sessionId: string; } export interface EvalResponse { @@ -32,100 +34,46 @@ export type EvalFunction = ( // Eval result types export interface EvalResult { - spanId: string; + sessionId: string; resultType: 'pass' | 'fail' | 'score'; scoreValue?: number; metadata?: { reasoning?: string; [key: string]: unknown }; timestamp: Date; } -export interface EvalRunnerConfig { - clickhouseHost: string; - clickhouseUser: string; - clickhousePassword: string; - spansTable?: string; - resultsTable?: string; -} - export default class EvalAPI { - private config: EvalRunnerConfig; - private client: ReturnType; - - constructor(config: EvalRunnerConfig) { - this.config = { - spansTable: 'spans', - resultsTable: 'eval_results', - ...config, - }; + private evalsDir: string; + private isBundled: boolean; - this.client = createClient({ - host: this.config.clickhouseHost, - username: this.config.clickhouseUser, - password: this.config.clickhousePassword, - }); - } + constructor(evalsDir?: string) { + // Check if we're running from bundled code (.agentuity directory) + const bundledDir = path.join(process.cwd(), '.agentuity', 'src', 'evals'); + const sourceDir = path.join(process.cwd(), 'src', 'evals'); + this.isBundled = fs.existsSync(bundledDir); - /** - * Fetch span data from ClickHouse and transform it for eval - */ - async fetchSpan(spanId: string): Promise { - internal.debug(`Fetching span ${spanId} from ClickHouse`); - - const query = ` - SELECT input, output - FROM ${this.config.spansTable} - WHERE spanId = {spanId:String} - LIMIT 1 - `; - - const result = await this.client.query({ - query, - query_params: { spanId }, - }); - - const response = await result.json<{ input: string; output: string }>(); - const row = response.data?.[0]; - - if (!row) { - throw new Error(`No span found for id ${spanId}`); - } + // Use .agentuity/src/evals for bundled code, src/evals for development + this.evalsDir = evalsDir || (this.isBundled ? bundledDir : sourceDir); - return { - input: row.input, - output: row.output, - spanId, - }; + internal.debug( + `EvalAPI initialized with evalsDir: ${this.evalsDir}, isBundled: ${this.isBundled}` + ); } /** - * Write eval result back to ClickHouse + * Load eval function from eval name */ - async writeResult(result: EvalResult): Promise { - internal.debug(`Writing eval result for span ${result.spanId}:`, result); - - await this.client.insert({ - table: this.config.resultsTable || 'eval_results', - values: [ - { - spanId: result.spanId, - resultType: result.resultType, - scoreValue: result.scoreValue ?? null, - metadata: result.metadata ? JSON.stringify(result.metadata) : null, - timestamp: result.timestamp.toISOString(), - }, - ], - format: 'JSONEachRow', - }); - } + async loadEval(evalName: string): Promise { + // For bundled code, eval files are .js in .agentuity/ + // For dev code, eval files are .ts in src/evals/ + const evalFile = this.isBundled ? `${evalName}.js` : evalName; + const evalPath = path.join(this.evalsDir, evalFile); - /** - * Load eval function from file path - */ - async loadEval(evalPath: string): Promise { internal.debug(`Loading eval function from ${evalPath}`); try { - const module = await import(evalPath); + // Convert to file URL for proper ESM import + const fileUrl = pathToFileURL(evalPath).href; + const module = await import(fileUrl); return module.default; } catch (error) { throw new Error( @@ -135,13 +83,21 @@ export default class EvalAPI { } /** - * Run eval on a span + * Run eval with input/output/sessionId */ - async runEval(spanId: string, evalPath: string): Promise { - internal.debug(`Running eval for span ${spanId} with function ${evalPath}`); - - // Load span data - const request = await this.fetchSpan(spanId); + async runEval( + evalName: string, + input: string, + output: string, + sessionId: string + ): Promise { + internal.debug(`Running eval ${evalName} for session ${sessionId}`); + + const request: EvalRequest = { + input, + output, + sessionId, + }; // Prepare response object let resultType: 'pass' | 'fail' | 'score' = 'fail'; @@ -169,47 +125,22 @@ export default class EvalAPI { const context: EvalContext = {}; // Load and run eval function - const evaluate = await this.loadEval(evalPath); + const evaluate = await this.loadEval(evalName); await evaluate(context, request, response); // Create result const result: EvalResult = { - spanId, + sessionId, resultType, scoreValue, metadata, timestamp: new Date(), }; - // Write result to database - await this.writeResult(result); - internal.debug( - `Eval complete for span ${spanId} -> ${resultType}${scoreValue !== undefined ? ` (${scoreValue})` : ''}` + `Eval complete for session ${sessionId} -> ${resultType}${scoreValue !== undefined ? ` (${scoreValue})` : ''}` ); return result; } - - /** - * Create the eval results table if it doesn't exist - */ - async createResultsTable(): Promise { - const createTableQuery = ` - CREATE TABLE IF NOT EXISTS ${this.config.resultsTable} ( - spanId String, - resultType String, - scoreValue Float64, - metadata String, - timestamp DateTime DEFAULT now() - ) ENGINE = MergeTree() - ORDER BY (spanId, timestamp) - `; - - await this.client.command({ - query: createTableQuery, - }); - - internal.debug(`Created eval results table: ${this.config.resultsTable}`); - } } diff --git a/src/index.ts b/src/index.ts index 2fdee73d..de84d9a3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,6 +17,15 @@ import PromptAPI from './apis/prompt'; import StreamAPIImpl from './apis/stream'; export { EmailAPI, DiscordAPI, EvalAPI, PatchPortal, PromptAPI, StreamAPIImpl }; +// Export eval types +export type { + EvalContext, + EvalFunction, + EvalRequest, + EvalResponse, + EvalResult, +} from './apis/eval'; + import { TeamsActivityHandler } from 'botbuilder'; import { run } from './autostart'; import { UnsupportedSlackPayload } from './io/slack'; diff --git a/src/server/bun.ts b/src/server/bun.ts index f339138d..cf56b40b 100644 --- a/src/server/bun.ts +++ b/src/server/bun.ts @@ -1,5 +1,7 @@ import type { ReadableStream } from 'node:stream/web'; import { context, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; +import EvalAPI from '../apis/eval'; +import { isIdle } from '../router/context'; import type { AgentResponseData, AgentWelcomeResult, @@ -17,7 +19,6 @@ import { shouldIgnoreStaticFile, toWelcomePrompt, } from './util'; -import { isIdle } from '../router/context'; const idleTimeout = 255; // expressed in seconds @@ -59,6 +60,7 @@ export class BunServer implements Server { const devmode = process.env.AGENTUITY_SDK_DEV_MODE === 'true'; const { sdkVersion, logger } = this.config; + const evalAPI = new EvalAPI(); const hostname = process.env.AGENTUITY_ENV === 'development' ? '127.0.0.1' : '0.0.0.0'; @@ -165,6 +167,43 @@ export class BunServer implements Server { const url = new URL(req.url); + // Handle eval routes + if (method === 'POST' && url.pathname.startsWith('/eval/')) { + const evalName = url.pathname.slice(6); // Remove '/eval/' + try { + const body = (await req.json()) as { + input: string; + output: string; + sessionId: string; + }; + const result = await evalAPI.runEval( + evalName, + body.input, + body.output, + body.sessionId + ); + return new Response(JSON.stringify(result), { + status: 200, + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + }, + }); + } catch (error) { + logger.error('eval error:', error); + return new Response( + JSON.stringify({ error: (error as Error).message }), + { + status: 500, + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + }, + } + ); + } + } + // Extract trace context from headers const extractedContext = extractTraceContextFromBunRequest(req); diff --git a/src/server/node.ts b/src/server/node.ts index 75193a2d..5ace8654 100644 --- a/src/server/node.ts +++ b/src/server/node.ts @@ -5,7 +5,9 @@ import { import { Readable } from 'node:stream'; import type { ReadableStream } from 'node:stream/web'; import { context, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; +import EvalAPI from '../apis/eval'; import type { Logger } from '../logger'; +import { isIdle } from '../router/context'; import type { AgentResponseData, AgentWelcomeResult } from '../types'; import { extractTraceContextFromNodeRequest, @@ -21,7 +23,6 @@ import { shouldIgnoreStaticFile, toWelcomePrompt, } from './util'; -import { isIdle } from '../router/context'; export const MAX_REQUEST_TIMEOUT = 60_000 * 10; @@ -104,6 +105,7 @@ export class NodeServer implements Server { async start(): Promise { const sdkVersion = this.sdkVersion; const devmode = process.env.AGENTUITY_SDK_DEV_MODE === 'true'; + const evalAPI = new EvalAPI(); this.server = createHttpServer(async (req, res) => { if (req.method === 'GET' && req.url === '/_health') { res.writeHead(200, { @@ -133,6 +135,41 @@ export class NodeServer implements Server { return; } + // Handle eval routes + if (req.method === 'POST' && req.url?.startsWith('/eval/')) { + const evalName = req.url.slice(6); // Remove '/eval/' + try { + let body = ''; + for await (const chunk of req) { + body += chunk; + } + const parsedBody = JSON.parse(body) as { + input: string; + output: string; + sessionId: string; + }; + const result = await evalAPI.runEval( + evalName, + parsedBody.input, + parsedBody.output, + parsedBody.sessionId + ); + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + }); + res.end(JSON.stringify(result)); + } catch (error) { + this.logger.error('eval error:', error); + res.writeHead(500, { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + }); + res.end(JSON.stringify({ error: (error as Error).message })); + } + return; + } + if (req.method === 'GET' && req.url === '/welcome') { const result: Record = {}; for (const route of this.routes) { From 788cc7d38499b7b001711dcd595a8e6802d59723 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Tue, 14 Oct 2025 13:46:22 -0400 Subject: [PATCH 14/26] merge conflict --- src/apis/eval.ts | 88 ++++++++++++++++++++++++++++++++++++++++++++-- src/index.ts | 10 +----- src/server/bun.ts | 2 +- src/server/node.ts | 2 +- 4 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/apis/eval.ts b/src/apis/eval.ts index 7b23bd2a..d899be2c 100644 --- a/src/apis/eval.ts +++ b/src/apis/eval.ts @@ -1,7 +1,9 @@ import fs from 'node:fs'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; +import { context, trace } from '@opentelemetry/api'; import { internal } from '../logger/internal'; +import { POST } from './api'; // Eval SDK types export interface EvalRequest { @@ -41,6 +43,25 @@ export interface EvalResult { timestamp: Date; } +// Request for storing eval run in DB +interface StoreEvalRunRequest { + projectId: string; + sessionId: string; + spanId: string; + result: EvalResult; + evalId?: string | null; + promptHash?: string | null; +} + +// Response from storing eval run +interface StoreEvalRunResponse { + success: boolean; + data?: { + id: string; + }; + message?: string; +} + export default class EvalAPI { private evalsDir: string; private isBundled: boolean; @@ -59,6 +80,50 @@ export default class EvalAPI { ); } + /** + * Store eval run result in database + */ + private async storeEvalRun( + evalName: string, + projectId: string, + sessionId: string, + spanId: string, + result: EvalResult + ): Promise { + try { + const payload: StoreEvalRunRequest = { + projectId, + sessionId, + spanId, + result, + evalId: evalName, + promptHash: null, // TODO: Add prompt hash when available + }; + + internal.debug(`Storing eval run for ${evalName} to /sdk/eval/run`); + + const resp = await POST( + '/sdk/eval/run', + JSON.stringify(payload), + { + 'Content-Type': 'application/json', + Accept: 'application/json', + } + ); + + if (resp.status === 200 || resp.status === 201) { + internal.debug(`Eval run stored successfully: ${resp.json?.data?.id}`); + } else { + internal.error( + `Failed to store eval run: ${resp.status} ${resp.json?.message}` + ); + } + } catch (error) { + internal.error(`Error storing eval run: ${error}`); + // Don't throw - we don't want to fail the eval if storage fails + } + } + /** * Load eval function from eval name */ @@ -93,6 +158,13 @@ export default class EvalAPI { ): Promise { internal.debug(`Running eval ${evalName} for session ${sessionId}`); + // Get project ID from environment + const projectId = process.env.AGENTUITY_CLOUD_PROJECT_ID || ''; + + // Get current span ID from OpenTelemetry context + const currentSpan = trace.getSpan(context.active()); + const spanId = currentSpan?.spanContext().spanId || ''; + const request: EvalRequest = { input, output, @@ -122,11 +194,11 @@ export default class EvalAPI { }, }; - const context: EvalContext = {}; + const evalContext: EvalContext = {}; // Load and run eval function const evaluate = await this.loadEval(evalName); - await evaluate(context, request, response); + await evaluate(evalContext, request, response); // Create result const result: EvalResult = { @@ -141,6 +213,18 @@ export default class EvalAPI { `Eval complete for session ${sessionId} -> ${resultType}${scoreValue !== undefined ? ` (${scoreValue})` : ''}` ); + // Store result in database (non-blocking) + if (projectId && spanId) { + // Don't await - fire and forget + this.storeEvalRun(evalName, projectId, sessionId, spanId, result).catch( + (error) => { + internal.error(`Failed to store eval run: ${error}`); + } + ); + } else { + internal.warn('Skipping eval storage - missing projectId or spanId'); + } + return result; } } diff --git a/src/index.ts b/src/index.ts index de84d9a3..55b4886a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,4 @@ +export * from './apis/eval'; export * from './logger'; export * from './server'; export * from './types'; @@ -17,15 +18,6 @@ import PromptAPI from './apis/prompt'; import StreamAPIImpl from './apis/stream'; export { EmailAPI, DiscordAPI, EvalAPI, PatchPortal, PromptAPI, StreamAPIImpl }; -// Export eval types -export type { - EvalContext, - EvalFunction, - EvalRequest, - EvalResponse, - EvalResult, -} from './apis/eval'; - import { TeamsActivityHandler } from 'botbuilder'; import { run } from './autostart'; import { UnsupportedSlackPayload } from './io/slack'; diff --git a/src/server/bun.ts b/src/server/bun.ts index cf56b40b..7e721ac0 100644 --- a/src/server/bun.ts +++ b/src/server/bun.ts @@ -167,7 +167,7 @@ export class BunServer implements Server { const url = new URL(req.url); - // Handle eval routes + // Handle eval endpoints if (method === 'POST' && url.pathname.startsWith('/eval/')) { const evalName = url.pathname.slice(6); // Remove '/eval/' try { diff --git a/src/server/node.ts b/src/server/node.ts index 5ace8654..6281a15b 100644 --- a/src/server/node.ts +++ b/src/server/node.ts @@ -135,7 +135,7 @@ export class NodeServer implements Server { return; } - // Handle eval routes + // Handle eval endpoints if (req.method === 'POST' && req.url?.startsWith('/eval/')) { const evalName = req.url.slice(6); // Remove '/eval/' try { From 19b93ed51fe377d6b49053fae4b1c9fae0cf4312 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Wed, 15 Oct 2025 14:47:44 -0400 Subject: [PATCH 15/26] added eval run --- .DS_Store | Bin 0 -> 8196 bytes package-lock.json | 6 +++--- package.json | 2 +- src/apis/api.ts | 12 +++++++++++- src/apis/eval.ts | 29 +++++++++++++++++------------ src/apis/prompt/index.ts | 19 +++++++++++++++---- src/server/bun.ts | 4 +++- src/server/node.ts | 4 +++- 8 files changed, 53 insertions(+), 23 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..04f3574e44b8976c393153791fc978f58b63c0ea GIT binary patch literal 8196 zcmeHMO>Yx15Pfb}Y6tt1kxE`{s;a7zmF5V z882Wvy9X|aim_zR#`~UU=6N@c9RTilJ9!Ko0_d@dy?2{UOOg9hcgkMV^MKZo4=gdm z6l>Pa4%#|Q0aL&fFa=BjQ{Z1vfNQptUdOuc?^>1*6xdZj_U?TQQQ{pg!~Of|qAJIWYRvocN*%EdziZ(YaG;zmR^07ZSS9GM z2|6Qs%F#c?IQOsxH^Us81b3a_#Jl1c^%mm<{VhRDejYuiT}r%|#+o^X{HNy3AhEv8O>RhmG zi9w!*h%?Z+noynB)TpATEwkqNQtV4Kig@=qZi0R&?|>SYd>-fYxJffp6+`qs(JPcm{-Rrm$~cmo!nM3b5$=w7 zr1y$Czt!iwBTpy#{Pw^lSXW^-x(d_U`p@^yiYaiD6xjE{9?Jgzr2G8;CZ%MtngXW4 zA1dIo(Z%SPBwf{8$g;}ktZ!JQ2(I>4DQu!20{r~z4?|w(gbJr { process.env.AGENTUITY_OBJECTSTORE_URL || process.env.AGENTUITY_TRANSPORT_URL; break; + case 'eval': + value = + process.env.AGENTUITY_EVAL_URL || process.env.AGENTUITY_TRANSPORT_URL; + break; default: break; } @@ -167,6 +176,7 @@ export async function send( init.keepalive = true; } + console.log('BOBBY!! url', url); const resp = await apiFetch(url, init); let json: K | null = null; switch (resp.status) { diff --git a/src/apis/eval.ts b/src/apis/eval.ts index d899be2c..ab5f3567 100644 --- a/src/apis/eval.ts +++ b/src/apis/eval.ts @@ -1,7 +1,6 @@ import fs from 'node:fs'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; -import { context, trace } from '@opentelemetry/api'; import { internal } from '../logger/internal'; import { POST } from './api'; @@ -100,15 +99,19 @@ export default class EvalAPI { promptHash: null, // TODO: Add prompt hash when available }; - internal.debug(`Storing eval run for ${evalName} to /sdk/eval/run`); + const url = '/evalrun/2025-03-17'; + internal.debug(`Storing eval run for ${evalName}`); const resp = await POST( - '/sdk/eval/run', + url, JSON.stringify(payload), { 'Content-Type': 'application/json', Accept: 'application/json', - } + }, + undefined, + undefined, + 'eval' ); if (resp.status === 200 || resp.status === 201) { @@ -148,23 +151,20 @@ export default class EvalAPI { } /** - * Run eval with input/output/sessionId + * Run eval with input/output/sessionId/spanId */ async runEval( evalName: string, input: string, output: string, - sessionId: string + sessionId: string, + spanId: string ): Promise { - internal.debug(`Running eval ${evalName} for session ${sessionId}`); + console.log(`Running eval ${evalName} for session ${sessionId}`); // Get project ID from environment const projectId = process.env.AGENTUITY_CLOUD_PROJECT_ID || ''; - // Get current span ID from OpenTelemetry context - const currentSpan = trace.getSpan(context.active()); - const spanId = currentSpan?.spanContext().spanId || ''; - const request: EvalRequest = { input, output, @@ -197,6 +197,8 @@ export default class EvalAPI { const evalContext: EvalContext = {}; // Load and run eval function + console.log('loading eval function'); + const evaluate = await this.loadEval(evalName); await evaluate(evalContext, request, response); @@ -209,12 +211,15 @@ export default class EvalAPI { timestamp: new Date(), }; - internal.debug( + console.log('BOBBY!! Eval result:', result); + + console.log( `Eval complete for session ${sessionId} -> ${resultType}${scoreValue !== undefined ? ` (${scoreValue})` : ''}` ); // Store result in database (non-blocking) if (projectId && spanId) { + console.log('writing eval result to database'); // Don't await - fire and forget this.storeEvalRun(evalName, projectId, sessionId, spanId, result).catch( (error) => { diff --git a/src/apis/prompt/index.ts b/src/apis/prompt/index.ts index e9a66487..c6d6cf79 100644 --- a/src/apis/prompt/index.ts +++ b/src/apis/prompt/index.ts @@ -88,7 +88,7 @@ export default class PromptAPI { // Method to load prompts dynamically (called by context) public async loadPrompts(): Promise { - internal.debug('loadPrompts() called'); + console.log('loadPrompts() called'); try { // Try multiple possible paths for the generated prompts let generatedModule: unknown; @@ -97,26 +97,37 @@ export default class PromptAPI { const possiblePaths = await this.resolveGeneratedPaths(); internal.debug('Trying absolute paths:', possiblePaths); + const attemptErrors: string[] = []; for (const possiblePath of possiblePaths) { internal.debug(' Checking:', possiblePath); try { await fs.access(possiblePath); + internal.debug(' โœ“ File exists:', possiblePath); + // Get file stats for cache-busting const stats = await fs.stat(possiblePath); const mtime = stats.mtime.getTime(); // Convert to file URL with cache-busting query param const fileUrl = pathToFileURL(possiblePath).href + `?t=${mtime}`; + internal.debug(' Importing from:', fileUrl); // Use ESM dynamic import instead of require generatedModule = await import(fileUrl); - internal.debug(' Successfully loaded from:', possiblePath); + internal.debug(' โœ“ Successfully loaded from:', possiblePath); break; - } catch {} + } catch (error) { + const errorMsg = + error instanceof Error ? error.message : String(error); + internal.debug(` โœ— Failed to load ${possiblePath}: ${errorMsg}`); + attemptErrors.push(`${possiblePath}: ${errorMsg}`); + } } if (!generatedModule) { - throw new Error('Generated prompts file not found'); + throw new Error( + `Generated prompts file not found. Tried:\n${attemptErrors.join('\n')}` + ); } // Type guard to ensure generatedModule has expected shape diff --git a/src/server/bun.ts b/src/server/bun.ts index 7e721ac0..97c92657 100644 --- a/src/server/bun.ts +++ b/src/server/bun.ts @@ -175,12 +175,14 @@ export class BunServer implements Server { input: string; output: string; sessionId: string; + spanId: string; }; const result = await evalAPI.runEval( evalName, body.input, body.output, - body.sessionId + body.sessionId, + body.spanId ); return new Response(JSON.stringify(result), { status: 200, diff --git a/src/server/node.ts b/src/server/node.ts index 6281a15b..d122960c 100644 --- a/src/server/node.ts +++ b/src/server/node.ts @@ -147,12 +147,14 @@ export class NodeServer implements Server { input: string; output: string; sessionId: string; + spanId: string; }; const result = await evalAPI.runEval( evalName, parsedBody.input, parsedBody.output, - parsedBody.sessionId + parsedBody.sessionId, + parsedBody.spanId ); res.writeHead(200, { 'Content-Type': 'application/json', From 4f0187c7ebd2a04d70055f140e7a4287ada1b052 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Thu, 16 Oct 2025 11:48:59 -0400 Subject: [PATCH 16/26] add all changes for review --- package.json | 2 +- src/apis/eval.ts | 188 +++++++++++++++++++++++++++++++-------------- src/server/bun.ts | 4 +- src/server/node.ts | 4 +- 4 files changed, 136 insertions(+), 62 deletions(-) diff --git a/package.json b/package.json index de7efb8d..6c67f14d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agentuity/sdk", - "version": "0.0.157", + "version": "0.0.158", "description": "The Agentuity SDK for NodeJS and Bun", "license": "Apache-2.0", "public": true, diff --git a/src/apis/eval.ts b/src/apis/eval.ts index ab5f3567..b15e0958 100644 --- a/src/apis/eval.ts +++ b/src/apis/eval.ts @@ -34,14 +34,40 @@ export type EvalFunction = ( ) => Promise; // Eval result types -export interface EvalResult { +interface BaseEvalRunResult { + evalId: string; sessionId: string; - resultType: 'pass' | 'fail' | 'score'; - scoreValue?: number; - metadata?: { reasoning?: string; [key: string]: unknown }; timestamp: Date; } +export type EvalRunResultBinary = BaseEvalRunResult & { + success: true; + passed: boolean; + metadata: { + reason: string; + [key: string]: any; + }; +}; + +export type EvalRunResultScore = BaseEvalRunResult & { + success: true; + score: number; // 0-1 range + metadata: { + reason: string; + [key: string]: any; + }; +}; + +export type EvalRunResultError = BaseEvalRunResult & { + success: false; + error: string; +}; + +export type EvalResult = + | EvalRunResultBinary + | EvalRunResultScore + | EvalRunResultError; + // Request for storing eval run in DB interface StoreEvalRunRequest { projectId: string; @@ -83,7 +109,7 @@ export default class EvalAPI { * Store eval run result in database */ private async storeEvalRun( - evalName: string, + evalId: string, projectId: string, sessionId: string, spanId: string, @@ -95,12 +121,14 @@ export default class EvalAPI { sessionId, spanId, result, - evalId: evalName, + evalId: evalId, promptHash: null, // TODO: Add prompt hash when available }; const url = '/evalrun/2025-03-17'; - internal.debug(`Storing eval run for ${evalName}`); + console.log('BOBBY!! Storing eval run with evalId:', evalId); + console.log('BOBBY!! Full payload:', JSON.stringify(payload, null, 2)); + internal.debug(`Storing eval run for eval ID: ${evalId}`); const resp = await POST( url, @@ -128,9 +156,11 @@ export default class EvalAPI { } /** - * Load eval function from eval name + * Load eval function and metadata from eval name */ - async loadEval(evalName: string): Promise { + async loadEval( + evalName: string + ): Promise<{ evalFn: EvalFunction; metadata?: { id: string } }> { // For bundled code, eval files are .js in .agentuity/ // For dev code, eval files are .ts in src/evals/ const evalFile = this.isBundled ? `${evalName}.js` : evalName; @@ -142,7 +172,10 @@ export default class EvalAPI { // Convert to file URL for proper ESM import const fileUrl = pathToFileURL(evalPath).href; const module = await import(fileUrl); - return module.default; + return { + evalFn: module.default, + metadata: module.metadata, + }; } catch (error) { throw new Error( `Failed to load eval function from ${evalPath}: ${error}` @@ -158,7 +191,8 @@ export default class EvalAPI { input: string, output: string, sessionId: string, - spanId: string + spanId: string, + evalId?: string ): Promise { console.log(`Running eval ${evalName} for session ${sessionId}`); @@ -171,65 +205,101 @@ export default class EvalAPI { sessionId, }; - // Prepare response object - let resultType: 'pass' | 'fail' | 'score' = 'fail'; - let scoreValue: number | undefined; - let metadata: { reasoning?: string; [key: string]: unknown } | undefined; - - const response: EvalResponse = { - pass: ( - value: boolean, - meta?: { reasoning?: string; [key: string]: unknown } - ) => { - resultType = value ? 'pass' : 'fail'; - metadata = meta; - }, - score: ( - val: number, - meta?: { reasoning?: string; [key: string]: unknown } - ) => { - resultType = 'score'; - scoreValue = val; - metadata = meta; - }, - }; + // Prepare response tracking + let result: EvalResult | null = null; + const timestamp = new Date(); const evalContext: EvalContext = {}; // Load and run eval function console.log('loading eval function'); - const evaluate = await this.loadEval(evalName); - await evaluate(evalContext, request, response); + try { + const { evalFn, metadata: evalMetadata } = await this.loadEval(evalName); + const finalEvalId = evalId || evalMetadata?.id || evalName; - // Create result - const result: EvalResult = { - sessionId, - resultType, - scoreValue, - metadata, - timestamp: new Date(), - }; + const response: EvalResponse = { + pass: ( + value: boolean, + meta?: { reasoning?: string; [key: string]: unknown } + ) => { + result = { + success: true, + passed: value, + metadata: { + reason: meta?.reasoning || '', + ...meta, + }, + evalId: finalEvalId, + sessionId, + timestamp, + }; + }, + score: ( + val: number, + meta?: { reasoning?: string; [key: string]: unknown } + ) => { + result = { + success: true, + score: val, + metadata: { + reason: meta?.reasoning || '', + ...meta, + }, + evalId: finalEvalId, + sessionId, + timestamp, + }; + }, + }; - console.log('BOBBY!! Eval result:', result); + await evalFn(evalContext, request, response); - console.log( - `Eval complete for session ${sessionId} -> ${resultType}${scoreValue !== undefined ? ` (${scoreValue})` : ''}` - ); + // If no result was set, create an error result + if (!result) { + result = { + success: false, + error: 'Eval function did not call res.pass() or res.score()', + evalId: finalEvalId, + sessionId, + timestamp, + }; + } + + console.log('BOBBY!! Eval result:', result); - // Store result in database (non-blocking) - if (projectId && spanId) { - console.log('writing eval result to database'); - // Don't await - fire and forget - this.storeEvalRun(evalName, projectId, sessionId, spanId, result).catch( - (error) => { + // Store result in database (non-blocking) + if (projectId && spanId) { + console.log( + 'writing eval result to database with evalId:', + finalEvalId + ); + this.storeEvalRun( + finalEvalId, + projectId, + sessionId, + spanId, + result + ).catch((error) => { internal.error(`Failed to store eval run: ${error}`); - } - ); - } else { - internal.warn('Skipping eval storage - missing projectId or spanId'); - } + }); + } else { + internal.warn('Skipping eval storage - missing projectId or spanId'); + } - return result; + return result; + } catch (error) { + // Return error result if eval function throws + result = { + success: false, + error: error instanceof Error ? error.message : String(error), + evalId: evalName, + sessionId, + timestamp, + }; + + console.log('BOBBY!! Eval error:', result); + return result; + } } } diff --git a/src/server/bun.ts b/src/server/bun.ts index 97c92657..a93c7b25 100644 --- a/src/server/bun.ts +++ b/src/server/bun.ts @@ -176,13 +176,15 @@ export class BunServer implements Server { output: string; sessionId: string; spanId: string; + evalId: string; }; const result = await evalAPI.runEval( evalName, body.input, body.output, body.sessionId, - body.spanId + body.spanId, + body.evalId ); return new Response(JSON.stringify(result), { status: 200, diff --git a/src/server/node.ts b/src/server/node.ts index d122960c..0d115f4f 100644 --- a/src/server/node.ts +++ b/src/server/node.ts @@ -148,13 +148,15 @@ export class NodeServer implements Server { output: string; sessionId: string; spanId: string; + evalId: string; }; const result = await evalAPI.runEval( evalName, parsedBody.input, parsedBody.output, parsedBody.sessionId, - parsedBody.spanId + parsedBody.spanId, + parsedBody.evalId ); res.writeHead(200, { 'Content-Type': 'application/json', From 42eeb6847c2e08d12380cc97d4bf7a78dd46de20 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Fri, 17 Oct 2025 10:12:26 -0400 Subject: [PATCH 17/26] cleaning up --- .DS_Store | Bin 8196 -> 0 bytes .gitignore | 3 +- src/apis/eval.ts | 230 ++++++++++++---------------------- src/apis/prompt/index.ts | 1 - src/server/bun.ts | 54 +++----- src/server/internal-routes.ts | 150 ++++++++++++++++++++++ src/server/node.ts | 53 +++----- 7 files changed, 265 insertions(+), 226 deletions(-) delete mode 100644 .DS_Store create mode 100644 src/server/internal-routes.ts diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 04f3574e44b8976c393153791fc978f58b63c0ea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMO>Yx15Pfb}Y6tt1kxE`{s;a7zmF5V z882Wvy9X|aim_zR#`~UU=6N@c9RTilJ9!Ko0_d@dy?2{UOOg9hcgkMV^MKZo4=gdm z6l>Pa4%#|Q0aL&fFa=BjQ{Z1vfNQptUdOuc?^>1*6xdZj_U?TQQQ{pg!~Of|qAJIWYRvocN*%EdziZ(YaG;zmR^07ZSS9GM z2|6Qs%F#c?IQOsxH^Us81b3a_#Jl1c^%mm<{VhRDejYuiT}r%|#+o^X{HNy3AhEv8O>RhmG zi9w!*h%?Z+noynB)TpATEwkqNQtV4Kig@=qZi0R&?|>SYd>-fYxJffp6+`qs(JPcm{-Rrm$~cmo!nM3b5$=w7 zr1y$Czt!iwBTpy#{Pw^lSXW^-x(d_U`p@^yiYaiD6xjE{9?Jgzr2G8;CZ%MtngXW4 zA1dIo(Z%SPBwf{8$g;}ktZ!JQ2(I>4DQu!20{r~z4?|w(gbJr + [key: string]: any; +}; -export type EvalFunction = ( - ctx: EvalContext, - req: EvalRequest, - res: EvalResponse -) => Promise; - -// Eval result types -interface BaseEvalRunResult { - evalId: string; - sessionId: string; - timestamp: Date; -} +type BaseEvalRunResult = { + success: boolean; + metadata?: EvalRunResultMetadata; +}; export type EvalRunResultBinary = BaseEvalRunResult & { success: true; passed: boolean; - metadata: { - reason: string; - [key: string]: any; - }; + metadata: EvalRunResultMetadata; }; export type EvalRunResultScore = BaseEvalRunResult & { success: true; score: number; // 0-1 range - metadata: { - reason: string; - [key: string]: any; - }; + metadata: EvalRunResultMetadata; }; export type EvalRunResultError = BaseEvalRunResult & { @@ -63,29 +54,37 @@ export type EvalRunResultError = BaseEvalRunResult & { error: string; }; -export type EvalResult = +export type EvalRunResult = | EvalRunResultBinary | EvalRunResultScore | EvalRunResultError; -// Request for storing eval run in DB -interface StoreEvalRunRequest { +export type CreateEvalRunRequest = { projectId: string; sessionId: string; spanId: string; - result: EvalResult; - evalId?: string | null; - promptHash?: string | null; -} + result: EvalRunResult; + evalId: string; + promptHash?: string; +}; -// Response from storing eval run -interface StoreEvalRunResponse { - success: boolean; - data?: { - id: string; - }; - message?: string; -} +type EvalFunction = ( + ctx: EvalContext, + req: EvalRequest, + res: EvalResponse +) => Promise; + +type CreateEvalRunResponse = + | { + success: true; + data: { + id: string; + }; + } + | { + success: false; + message: string; + }; export default class EvalAPI { private evalsDir: string; @@ -105,56 +104,6 @@ export default class EvalAPI { ); } - /** - * Store eval run result in database - */ - private async storeEvalRun( - evalId: string, - projectId: string, - sessionId: string, - spanId: string, - result: EvalResult - ): Promise { - try { - const payload: StoreEvalRunRequest = { - projectId, - sessionId, - spanId, - result, - evalId: evalId, - promptHash: null, // TODO: Add prompt hash when available - }; - - const url = '/evalrun/2025-03-17'; - console.log('BOBBY!! Storing eval run with evalId:', evalId); - console.log('BOBBY!! Full payload:', JSON.stringify(payload, null, 2)); - internal.debug(`Storing eval run for eval ID: ${evalId}`); - - const resp = await POST( - url, - JSON.stringify(payload), - { - 'Content-Type': 'application/json', - Accept: 'application/json', - }, - undefined, - undefined, - 'eval' - ); - - if (resp.status === 200 || resp.status === 201) { - internal.debug(`Eval run stored successfully: ${resp.json?.data?.id}`); - } else { - internal.error( - `Failed to store eval run: ${resp.status} ${resp.json?.message}` - ); - } - } catch (error) { - internal.error(`Error storing eval run: ${error}`); - // Don't throw - we don't want to fail the eval if storage fails - } - } - /** * Load eval function and metadata from eval name */ @@ -192,10 +141,10 @@ export default class EvalAPI { output: string, sessionId: string, spanId: string, - evalId?: string - ): Promise { - console.log(`Running eval ${evalName} for session ${sessionId}`); - + evalId: string, + promptHash?: string + ): Promise { + internal.debug(`Running eval ${evalName} for session ${sessionId}`); // Get project ID from environment const projectId = process.env.AGENTUITY_CLOUD_PROJECT_ID || ''; @@ -205,50 +154,54 @@ export default class EvalAPI { sessionId, }; - // Prepare response tracking - let result: EvalResult | null = null; - const timestamp = new Date(); - + let createEvalRunRequest: CreateEvalRunRequest | null = null; const evalContext: EvalContext = {}; // Load and run eval function - console.log('loading eval function'); + internal.debug('loading eval function'); try { - const { evalFn, metadata: evalMetadata } = await this.loadEval(evalName); - const finalEvalId = evalId || evalMetadata?.id || evalName; + const { evalFn } = await this.loadEval(evalName); const response: EvalResponse = { pass: ( value: boolean, meta?: { reasoning?: string; [key: string]: unknown } ) => { - result = { - success: true, - passed: value, - metadata: { - reason: meta?.reasoning || '', - ...meta, - }, - evalId: finalEvalId, + createEvalRunRequest = { + projectId, sessionId, - timestamp, + spanId, + result: { + success: true, + passed: value, + metadata: { + reason: meta?.reasoning || '', + ...meta, + }, + }, + evalId, + promptHash, }; }, score: ( val: number, meta?: { reasoning?: string; [key: string]: unknown } ) => { - result = { - success: true, - score: val, - metadata: { - reason: meta?.reasoning || '', - ...meta, - }, - evalId: finalEvalId, + createEvalRunRequest = { + projectId, sessionId, - timestamp, + spanId, + result: { + success: true, + score: val, + metadata: { + reason: meta?.reasoning || '', + ...meta, + }, + }, + evalId, + promptHash, }; }, }; @@ -256,50 +209,29 @@ export default class EvalAPI { await evalFn(evalContext, request, response); // If no result was set, create an error result - if (!result) { - result = { - success: false, - error: 'Eval function did not call res.pass() or res.score()', - evalId: finalEvalId, - sessionId, - timestamp, - }; + if (!createEvalRunRequest) { + throw new Error('Eval function did not call res.pass() or res.score()'); } - console.log('BOBBY!! Eval result:', result); + const resp = await POST( + `/_agentuity/eval/${evalId}/runs`, + JSON.stringify(createEvalRunRequest), + { + 'Content-Type': 'application/json', + } + ); - // Store result in database (non-blocking) - if (projectId && spanId) { - console.log( - 'writing eval result to database with evalId:', - finalEvalId - ); - this.storeEvalRun( - finalEvalId, - projectId, - sessionId, - spanId, - result - ).catch((error) => { - internal.error(`Failed to store eval run: ${error}`); - }); - } else { - internal.warn('Skipping eval storage - missing projectId or spanId'); + if (!resp.json?.success) { + throw new Error('Failed to create eval run'); } - return result; + return resp.json; } catch (error) { // Return error result if eval function throws - result = { + return { success: false, - error: error instanceof Error ? error.message : String(error), - evalId: evalName, - sessionId, - timestamp, + message: error instanceof Error ? error.message : String(error), }; - - console.log('BOBBY!! Eval error:', result); - return result; } } } diff --git a/src/apis/prompt/index.ts b/src/apis/prompt/index.ts index c6d6cf79..ca2a52d6 100644 --- a/src/apis/prompt/index.ts +++ b/src/apis/prompt/index.ts @@ -88,7 +88,6 @@ export default class PromptAPI { // Method to load prompts dynamically (called by context) public async loadPrompts(): Promise { - console.log('loadPrompts() called'); try { // Try multiple possible paths for the generated prompts let generatedModule: unknown; diff --git a/src/server/bun.ts b/src/server/bun.ts index a93c7b25..7e9869b0 100644 --- a/src/server/bun.ts +++ b/src/server/bun.ts @@ -1,12 +1,12 @@ import type { ReadableStream } from 'node:stream/web'; import { context, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; -import EvalAPI from '../apis/eval'; import { isIdle } from '../router/context'; import type { AgentResponseData, AgentWelcomeResult, ReadableDataType, } from '../types'; +import { InternalRoutesHandler } from './internal-routes'; import { extractTraceContextFromBunRequest, injectTraceContextToHeaders, @@ -60,7 +60,7 @@ export class BunServer implements Server { const devmode = process.env.AGENTUITY_SDK_DEV_MODE === 'true'; const { sdkVersion, logger } = this.config; - const evalAPI = new EvalAPI(); + const internalRoutes = new InternalRoutesHandler(logger); const hostname = process.env.AGENTUITY_ENV === 'development' ? '127.0.0.1' : '0.0.0.0'; @@ -167,44 +167,18 @@ export class BunServer implements Server { const url = new URL(req.url); - // Handle eval endpoints - if (method === 'POST' && url.pathname.startsWith('/eval/')) { - const evalName = url.pathname.slice(6); // Remove '/eval/' - try { - const body = (await req.json()) as { - input: string; - output: string; - sessionId: string; - spanId: string; - evalId: string; - }; - const result = await evalAPI.runEval( - evalName, - body.input, - body.output, - body.sessionId, - body.spanId, - body.evalId - ); - return new Response(JSON.stringify(result), { - status: 200, - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - }, - }); - } catch (error) { - logger.error('eval error:', error); - return new Response( - JSON.stringify({ error: (error as Error).message }), - { - status: 500, - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - }, - } - ); + // Handle internal routes first + if (url.pathname.startsWith('/_agentuity/')) { + const internalResponse = await internalRoutes.handleInternalRoute({ + method, + url: url.pathname, + headers: req.headers.toJSON(), + body: req.body as unknown as ReadableStream, + request: getRequestFromHeaders(req.headers.toJSON(), ''), + setTimeout: (_val: number) => void 0, + }); + if (internalResponse) { + return internalResponse; } } diff --git a/src/server/internal-routes.ts b/src/server/internal-routes.ts new file mode 100644 index 00000000..fe1ce938 --- /dev/null +++ b/src/server/internal-routes.ts @@ -0,0 +1,150 @@ +import EvalAPI from '../apis/eval'; +import type { Logger } from '../logger'; +import type { ServerRequest } from './types'; + +/** + * Handles internal /_agentuity routes + * These routes are used internally by the SDK and should not be exposed to users + */ +export class InternalRoutesHandler { + private logger: Logger; + + constructor(logger: Logger) { + this.logger = logger; + } + + /** + * Handles internal eval routes: /_agentuity/eval/:evalId (POST) + */ + async handleEval(req: ServerRequest): Promise { + if (req.method !== 'POST') { + return new Response('Method Not Allowed', { status: 405 }); + } + + try { + // Parse the eval ID from the URL path + // Expected format: /_agentuity/eval/{evalId} + const pathParts = req.url.split('/'); + const evalIdIndex = pathParts.indexOf('eval'); + if (evalIdIndex === -1 || evalIdIndex + 1 >= pathParts.length) { + return new Response('Invalid eval ID in path', { status: 400 }); + } + const evalId = pathParts[evalIdIndex + 1]; + + // Parse request body + const body = await this.parseRequestBody(req); + if (!body) { + return new Response('Invalid request body', { status: 400 }); + } + + // Type assertion for the body + const evalBody = body as { + input: string; + output: string; + sessionId: string; + spanId: string; + evalName: string; + }; + + // Validate required fields + if ( + !evalBody.input || + !evalBody.output || + !evalBody.sessionId || + !evalBody.spanId || + !evalBody.evalName + ) { + return new Response('Missing required fields', { status: 400 }); + } + + const evalAPI = new EvalAPI(); + + // Run the eval + const result = await evalAPI.runEval( + evalBody.evalName, // evalName (slug) + evalBody.input, + evalBody.output, + evalBody.sessionId, + evalBody.spanId, + evalId // evalId from URL path + ); + + return new Response(JSON.stringify(result), { + status: 200, + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + }, + }); + } catch (error) { + this.logger.error('Internal eval route error:', error); + return new Response(JSON.stringify({ error: (error as Error).message }), { + status: 500, + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + }, + }); + } + } + + /** + * Checks if a request is for an internal route + */ + isInternalRoute(url: string): boolean { + return url.startsWith('/_agentuity/'); + } + + /** + * Routes internal requests to appropriate handlers + */ + async handleInternalRoute(req: ServerRequest): Promise { + if (!this.isInternalRoute(req.url)) { + return null; + } + + // Handle eval routes + if (req.url.startsWith('/_agentuity/eval/')) { + return this.handleEval(req); + } + + // Add more internal routes here as needed + // e.g., /_agentuity/analytics, /_agentuity/metrics, etc. + + return new Response('Internal route not found', { status: 404 }); + } + + /** + * Parses request body from different request types + */ + private async parseRequestBody(req: ServerRequest): Promise { + if (req.body) { + // Handle ReadableStream body + if (req.body instanceof ReadableStream) { + const reader = req.body.getReader(); + const chunks: Uint8Array[] = []; + let done = false; + + while (!done) { + const { value, done: readerDone } = await reader.read(); + done = readerDone; + if (value) { + chunks.push(value); + } + } + + const bodyText = new TextDecoder().decode( + new Uint8Array( + chunks.reduce((acc, chunk) => { + acc.push(...chunk); + return acc; + }, [] as number[]) + ) + ); + return JSON.parse(bodyText); + } + } + + return null; + } +} diff --git a/src/server/node.ts b/src/server/node.ts index 0d115f4f..f3c0d3da 100644 --- a/src/server/node.ts +++ b/src/server/node.ts @@ -5,10 +5,10 @@ import { import { Readable } from 'node:stream'; import type { ReadableStream } from 'node:stream/web'; import { context, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; -import EvalAPI from '../apis/eval'; import type { Logger } from '../logger'; import { isIdle } from '../router/context'; import type { AgentResponseData, AgentWelcomeResult } from '../types'; +import { InternalRoutesHandler } from './internal-routes'; import { extractTraceContextFromNodeRequest, injectTraceContextToHeaders, @@ -105,7 +105,7 @@ export class NodeServer implements Server { async start(): Promise { const sdkVersion = this.sdkVersion; const devmode = process.env.AGENTUITY_SDK_DEV_MODE === 'true'; - const evalAPI = new EvalAPI(); + const internalRoutes = new InternalRoutesHandler(this.logger); this.server = createHttpServer(async (req, res) => { if (req.method === 'GET' && req.url === '/_health') { res.writeHead(200, { @@ -135,43 +135,26 @@ export class NodeServer implements Server { return; } - // Handle eval endpoints - if (req.method === 'POST' && req.url?.startsWith('/eval/')) { - const evalName = req.url.slice(6); // Remove '/eval/' - try { - let body = ''; - for await (const chunk of req) { - body += chunk; - } - const parsedBody = JSON.parse(body) as { - input: string; - output: string; - sessionId: string; - spanId: string; - evalId: string; - }; - const result = await evalAPI.runEval( - evalName, - parsedBody.input, - parsedBody.output, - parsedBody.sessionId, - parsedBody.spanId, - parsedBody.evalId - ); - res.writeHead(200, { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - }); - res.end(JSON.stringify(result)); - } catch (error) { - this.logger.error('eval error:', error); - res.writeHead(500, { + // Handle internal routes first + if (req.url?.startsWith('/_agentuity/')) { + const body = await this.getBufferAsStream(req); + const internalResponse = await internalRoutes.handleInternalRoute({ + method: req.method || 'GET', + url: req.url, + headers: this.getHeaders(req), + body, + request: getRequestFromHeaders(this.getHeaders(req), ''), + setTimeout: (_val: number) => void 0, + }); + if (internalResponse) { + const responseBody = await internalResponse.text(); + res.writeHead(internalResponse.status, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*', }); - res.end(JSON.stringify({ error: (error as Error).message })); + res.end(responseBody); + return; } - return; } if (req.method === 'GET' && req.url === '/welcome') { From 073adeb6691e3b1277c0320e695ec556d73cf4e4 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Mon, 20 Oct 2025 15:21:06 -0400 Subject: [PATCH 18/26] added eval job scheduler --- src/apis/eval.ts | 70 +++++++++++------ src/apis/evaljobscheduler.ts | 148 +++++++++++++++++++++++++++++++++++ src/index.ts | 11 ++- 3 files changed, 206 insertions(+), 23 deletions(-) create mode 100644 src/apis/evaljobscheduler.ts diff --git a/src/apis/eval.ts b/src/apis/eval.ts index 2ef1c978..96981693 100644 --- a/src/apis/eval.ts +++ b/src/apis/eval.ts @@ -28,7 +28,7 @@ export interface EvalContext { } export type EvalRunResultMetadata = { reason: string; - // biome-ignore lint/suspicious/noExplicitAny: + // biome-ignore lint/suspicious/noExplicitAny: metadata can contain any type of data [key: string]: any; }; @@ -105,30 +105,55 @@ export default class EvalAPI { } /** - * Load eval function and metadata from eval name + * Load eval function and metadata by ID + * Scans through all eval files to find the one with matching ID */ - async loadEval( - evalName: string - ): Promise<{ evalFn: EvalFunction; metadata?: { id: string } }> { - // For bundled code, eval files are .js in .agentuity/ - // For dev code, eval files are .ts in src/evals/ - const evalFile = this.isBundled ? `${evalName}.js` : evalName; - const evalPath = path.join(this.evalsDir, evalFile); - - internal.debug(`Loading eval function from ${evalPath}`); + async loadEvalById(evalId: string): Promise<{ + evalFn: EvalFunction; + metadata?: { id: string; slug: string; name: string; description: string }; + }> { + internal.debug(`Loading eval by ID: ${evalId}`); try { - // Convert to file URL for proper ESM import - const fileUrl = pathToFileURL(evalPath).href; - const module = await import(fileUrl); - return { - evalFn: module.default, - metadata: module.metadata, - }; + // Get all files in the evals directory + const files = fs.readdirSync(this.evalsDir); + + for (const file of files) { + // Skip index files and non-eval files + if (file === 'index.ts' || file === 'index.js') { + continue; + } + + // Check file extension based on bundled state + const expectedExt = this.isBundled ? '.js' : '.ts'; + if (!file.endsWith(expectedExt)) { + continue; + } + + const filePath = path.join(this.evalsDir, file); + + try { + // Convert to file URL for proper ESM import + const fileUrl = pathToFileURL(filePath).href; + const module = await import(fileUrl); + + // Check if this module has the matching ID + if (module.metadata && module.metadata.id === evalId) { + internal.debug(`Found eval with ID ${evalId} in file ${file}`); + return { + evalFn: module.default, + metadata: module.metadata, + }; + } + } catch (error) { + // Skip files that can't be imported (might not be eval files) + internal.debug(`Skipping file ${file} due to import error: ${error}`); + } + } + + throw new Error(`No eval found with ID: ${evalId}`); } catch (error) { - throw new Error( - `Failed to load eval function from ${evalPath}: ${error}` - ); + throw new Error(`Failed to load eval by ID ${evalId}: ${error}`); } } @@ -161,7 +186,8 @@ export default class EvalAPI { internal.debug('loading eval function'); try { - const { evalFn } = await this.loadEval(evalName); + // Try to load by ID first, fallback to name + const { evalFn } = await this.loadEvalById(evalId); const response: EvalResponse = { pass: ( diff --git a/src/apis/evaljobscheduler.ts b/src/apis/evaljobscheduler.ts new file mode 100644 index 00000000..2ee5b21e --- /dev/null +++ b/src/apis/evaljobscheduler.ts @@ -0,0 +1,148 @@ +import { internal } from '../logger/internal'; +import type { PromptAttributes } from '../utils/promptMetadata'; + +// Global instance storage to ensure true singleton across all module contexts +declare global { + var __evalJobSchedulerInstance: EvalJobScheduler | undefined; +} + +export interface PendingEvalJob { + spanId: string; + sessionId: string; + promptMetadata: PromptAttributes[]; + output?: string; + createdAt: string; +} + +export interface JobFilter { + sessionId?: string; +} + +/** + * Singleton class for EvalJobScheduler + */ +export default class EvalJobScheduler { + private pendingJobs: Map = new Map(); + private instanceId: string; + + private constructor() { + // Private constructor to prevent direct instantiation + this.instanceId = `EvalJobScheduler-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + internal.debug( + '๐Ÿ—๏ธ EvalJobScheduler constructor called, instanceId:', + this.instanceId + ); + } + + /** + * Get the singleton instance of EvalJobScheduler + */ + public static async getInstance(): Promise { + internal.debug('๐Ÿ” EvalJobScheduler.getInstance() called'); + internal.debug( + '๐Ÿ” globalThis.__evalJobSchedulerInstance exists:', + !!globalThis.__evalJobSchedulerInstance + ); + + if (!globalThis.__evalJobSchedulerInstance) { + globalThis.__evalJobSchedulerInstance = new EvalJobScheduler(); + internal.debug( + '๐Ÿ†• Created new EvalJobScheduler instance, ID:', + globalThis.__evalJobSchedulerInstance.instanceId + ); + } else { + internal.debug( + 'โ™ป๏ธ Returning existing EvalJobScheduler instance, ID:', + globalThis.__evalJobSchedulerInstance.instanceId + ); + } + return globalThis.__evalJobSchedulerInstance; + } + + /** + * Create a new eval job + */ + public createJob( + spanId: string, + sessionId: string, + promptMetadata: PromptAttributes[] + ): string { + internal.debug('๐Ÿ” EvalJobScheduler.createJob() called with:', { + spanId, + sessionId, + promptMetadataCount: promptMetadata.length, + }); + + const now = new Date().toISOString(); + + // Check if job already exists + if (this.pendingJobs.has(spanId)) { + internal.debug('โš ๏ธ Job already exists, overwriting:', spanId); + } + + const job: PendingEvalJob = { + spanId, + sessionId, + promptMetadata, + createdAt: now, + }; + + this.pendingJobs.set(spanId, job); + internal.debug('โœ… Job created successfully:', spanId); + internal.debug('๐Ÿ“Š Total jobs:', this.pendingJobs.size); + + return spanId; + } + + /** + * Remove a job + */ + public removeJob(spanId: string): boolean { + internal.debug('๐Ÿ” EvalJobScheduler.removeJob() called with:', spanId); + const removed = this.pendingJobs.delete(spanId); + internal.debug('๐Ÿ” Job removed:', removed); + return removed; + } + + /** + * Get jobs with optional filtering + */ + public getJobs(filter?: JobFilter): PendingEvalJob[] { + internal.debug('๐Ÿ” EvalJobScheduler.getJobs() called with filter:', filter); + + let jobs = Array.from(this.pendingJobs.values()); + + if (filter?.sessionId) { + jobs = jobs.filter((job) => job.sessionId === filter.sessionId); + } + + internal.debug('๐Ÿ” Found jobs:', jobs.length); + return jobs; + } + + /** + * Print out the whole state of the EvalJobScheduler + */ + public printState(): void { + internal.debug('๐Ÿ” EvalJobScheduler.printState() called'); + internal.debug('๐Ÿ” Instance ID:', this.instanceId); + internal.debug('๐Ÿ” EvalJobScheduler State:'); + internal.debug('๐Ÿ“Š Total jobs:', this.pendingJobs.size); + internal.debug('๐Ÿ“‹ Job IDs:', Array.from(this.pendingJobs.keys())); + internal.debug( + '๐Ÿ“ฆ All jobs:', + JSON.stringify(Array.from(this.pendingJobs.values()), null, 2) + ); + internal.debug( + '๐Ÿ” Global instance check:', + globalThis.__evalJobSchedulerInstance === this + ); + } + + /** + * Get the instance ID + */ + public getInstanceId(): string { + return this.instanceId; + } +} diff --git a/src/index.ts b/src/index.ts index 55b4886a..8206c275 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,10 +13,19 @@ import DiscordAPI from './apis/discord'; // Export APIs import EmailAPI from './apis/email'; import EvalAPI from './apis/eval'; +import EvalJobScheduler from './apis/evaljobscheduler'; import PatchPortal from './apis/patchportal'; import PromptAPI from './apis/prompt'; import StreamAPIImpl from './apis/stream'; -export { EmailAPI, DiscordAPI, EvalAPI, PatchPortal, PromptAPI, StreamAPIImpl }; +export { + EmailAPI, + DiscordAPI, + EvalAPI, + EvalJobScheduler, + PatchPortal, + PromptAPI, + StreamAPIImpl, +}; import { TeamsActivityHandler } from 'botbuilder'; import { run } from './autostart'; From e5d730e7b7921b5b17eb53bf73253233fd381b2d Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Mon, 20 Oct 2025 16:54:58 -0400 Subject: [PATCH 19/26] added eval job scheduler --- src/apis/eval.ts | 120 +++++++++++++++++++++++++++ src/apis/evaljobscheduler.ts | 29 ++++++- src/router/context.ts | 151 +++++++++++++++++++++++++++++++++- src/router/router.ts | 10 ++- src/server/bun.ts | 8 ++ src/server/internal-routes.ts | 121 +-------------------------- src/server/node.ts | 5 ++ 7 files changed, 320 insertions(+), 124 deletions(-) diff --git a/src/apis/eval.ts b/src/apis/eval.ts index 96981693..b426d37e 100644 --- a/src/apis/eval.ts +++ b/src/apis/eval.ts @@ -260,4 +260,124 @@ export default class EvalAPI { }; } } + + /** + * Load eval metadata map from eval files (slug -> ID mapping) + * Scans through all eval files to find metadata and build mapping + */ + async loadEvalMetadataMap(): Promise> { + internal.info(`๐Ÿ” Loading eval metadata map from: ${this.evalsDir}`); + + // Check if evals directory exists + if (!fs.existsSync(this.evalsDir)) { + internal.info(`๐Ÿ“ Evals directory not found: ${this.evalsDir}`); + return new Map(); + } + + const files = fs.readdirSync(this.evalsDir); + const slugToIDMap = new Map(); + let processedFiles = 0; + + internal.info(`๐Ÿ“‚ Scanning ${files.length} files in evals directory`); + + for (const file of files) { + const ext = path.extname(file); + if ( + file === 'index.ts' || + file === 'index.js' || + (ext !== '.ts' && ext !== '.js') + ) { + internal.debug(`โญ๏ธ Skipping file: ${file}`); + continue; + } + + const filePath = path.join(this.evalsDir, file); + processedFiles++; + + try { + const content = fs.readFileSync(filePath, 'utf-8'); + const metadata = this.parseEvalMetadata(content); + + if (metadata && metadata.slug && metadata.id) { + slugToIDMap.set(metadata.slug, metadata.id); + internal.info( + `โœ… Mapped eval slug '${metadata.slug}' to ID '${metadata.id}' from ${file}` + ); + } else { + internal.debug(`โš ๏ธ No valid metadata found in ${file}`); + } + } catch (error) { + internal.warn(`โŒ Failed to parse metadata from ${file}: ${error}`); + } + } + + internal.info( + `๐Ÿ“š Loaded ${slugToIDMap.size} eval mappings from ${processedFiles} files` + ); + return slugToIDMap; + } + + /** + * Parse eval metadata from file content + * Similar to CLI's ParseEvalMetadata but in TypeScript + */ + private parseEvalMetadata( + content: string + ): { id: string; slug: string; name: string; description: string } | null { + // Find the metadata export pattern + const metadataRegex = /export\s+const\s+metadata\s*=\s*\{/; + const metadataMatch = content.match(metadataRegex); + if (!metadataMatch) { + return null; + } + + // Find the opening brace position + const braceStart = metadataMatch.index! + metadataMatch[0].length - 1; + if (braceStart >= content.length || content[braceStart] !== '{') { + return null; + } + + // Count braces to find the matching closing brace + let braceCount = 0; + let braceEnd = -1; + for (let i = braceStart; i < content.length; i++) { + if (content[i] === '{') { + braceCount++; + } else if (content[i] === '}') { + braceCount--; + if (braceCount === 0) { + braceEnd = i; + break; + } + } + } + + if (braceEnd === -1) { + return null; + } + + // Extract the object content + const objectContent = content.slice(braceStart, braceEnd + 1); + + // Replace single quotes with double quotes for valid JSON + let jsonStr = objectContent.replace(/'([^']*)'/g, '"$1"'); + + // Clean up the JSON string + jsonStr = jsonStr.replace(/\s+/g, ' '); + jsonStr = jsonStr.replace(/\s*{\s*/g, '{'); + jsonStr = jsonStr.replace(/\s*}\s*/g, '}'); + jsonStr = jsonStr.replace(/\s*:\s*/g, ':'); + jsonStr = jsonStr.replace(/\s*,\s*/g, ','); + jsonStr = jsonStr.replace(/,\s*}/g, '}'); + + // Quote the object keys + jsonStr = jsonStr.replace(/(\w+):/g, '"$1":'); + + try { + return JSON.parse(jsonStr); + } catch (error) { + internal.debug(`Failed to parse metadata JSON: ${error}`); + return null; + } + } } diff --git a/src/apis/evaljobscheduler.ts b/src/apis/evaljobscheduler.ts index 2ee5b21e..5602573d 100644 --- a/src/apis/evaljobscheduler.ts +++ b/src/apis/evaljobscheduler.ts @@ -80,6 +80,15 @@ export default class EvalJobScheduler { internal.debug('โš ๏ธ Job already exists, overwriting:', spanId); } + // Count total evals across all prompt metadata + const totalEvals = promptMetadata.reduce( + (count, meta) => count + (meta.evals?.length || 0), + 0 + ); + internal.info( + `๐Ÿ“ฆ Creating eval job ${spanId} for session ${sessionId} with ${totalEvals} evals` + ); + const job: PendingEvalJob = { spanId, sessionId, @@ -108,15 +117,31 @@ export default class EvalJobScheduler { * Get jobs with optional filtering */ public getJobs(filter?: JobFilter): PendingEvalJob[] { - internal.debug('๐Ÿ” EvalJobScheduler.getJobs() called with filter:', filter); + internal.info('๐Ÿ” EvalJobScheduler.getJobs() called with filter:', filter); + internal.info('๐Ÿ“Š Total pending jobs in scheduler:', this.pendingJobs.size); let jobs = Array.from(this.pendingJobs.values()); if (filter?.sessionId) { jobs = jobs.filter((job) => job.sessionId === filter.sessionId); + internal.info( + `๐Ÿ” Filtered jobs for session ${filter.sessionId}:`, + jobs.length + ); } - internal.debug('๐Ÿ” Found jobs:', jobs.length); + internal.info('๐Ÿ“‹ Returning jobs:', jobs.length); + if (jobs.length > 0) { + jobs.forEach((job, index) => { + internal.info(`๐Ÿ“ฆ Job ${index + 1}:`, { + spanId: job.spanId, + sessionId: job.sessionId, + promptMetadataCount: job.promptMetadata?.length || 0, + hasOutput: !!job.output, + createdAt: job.createdAt, + }); + }); + } return jobs; } diff --git a/src/router/context.ts b/src/router/context.ts index c8dd8828..4b4d2d47 100644 --- a/src/router/context.ts +++ b/src/router/context.ts @@ -1,9 +1,11 @@ import { context, SpanStatusCode, - trace, type Tracer, + trace, } from '@opentelemetry/api'; +import EvalAPI from '../apis/eval'; +import EvalJobScheduler from '../apis/evaljobscheduler'; import { markSessionCompleted } from '../apis/session'; import type { Logger } from '../logger'; @@ -68,19 +70,33 @@ export default class AgentContextWaitUntilHandler { } public async waitUntilAll(logger: Logger, sessionId: string): Promise { + logger.info(`๐Ÿ” waitUntilAll() called for session ${sessionId}`); + if (this.hasCalledWaitUntilAll) { throw new Error('waitUntilAll can only be called once per instance'); } this.hasCalledWaitUntilAll = true; if (this.promises.length === 0) { + logger.info('๐Ÿ“ญ No promises to wait for, executing evals directly'); + // Execute evals even if no promises + await this.executeEvalsForSession(logger, sessionId); return; } + + logger.info( + `โณ Waiting for ${this.promises.length} promises to complete...` + ); try { // Promises are already executing, just wait for them to complete await Promise.all(this.promises); const duration = Date.now() - (this.started as number); + logger.info('โœ… All promises completed, marking session completed'); await markSessionCompleted(sessionId, duration); + + // Execute evals after session completion + logger.info('๐Ÿš€ Starting eval execution after session completion'); + await this.executeEvalsForSession(logger, sessionId); } catch (ex) { logger.error('error sending session completed', ex); } finally { @@ -88,4 +104,137 @@ export default class AgentContextWaitUntilHandler { this.promises.length = 0; } } + + /** + * Execute evals for the completed session + */ + private async executeEvalsForSession( + logger: Logger, + sessionId: string + ): Promise { + try { + logger.info(`๐Ÿ” Starting eval execution for session ${sessionId}`); + + // Get pending eval jobs for this session + logger.info('๐Ÿ” Getting EvalJobScheduler instance...'); + const evalJobScheduler = await EvalJobScheduler.getInstance(); + logger.info('โœ… EvalJobScheduler instance obtained'); + + logger.info(`๐Ÿ” Querying jobs for session ${sessionId}...`); + const jobs = evalJobScheduler.getJobs({ sessionId }); + + if (jobs.length === 0) { + logger.info(`๐Ÿ“ญ No eval jobs found for session ${sessionId}`); + return; + } + + logger.info(`๐Ÿ“‹ Found ${jobs.length} eval jobs for session ${sessionId}`); + + // Load eval metadata map + logger.info('๐Ÿ”ง Loading eval metadata map...'); + const evalAPI = new EvalAPI(); + const evalMetadataMap = await evalAPI.loadEvalMetadataMap(); + logger.info(`๐Ÿ“š Loaded ${evalMetadataMap.size} eval mappings`); + + // Execute evals for each job + let totalEvalsExecuted = 0; + for (let i = 0; i < jobs.length; i++) { + const job = jobs[i]; + logger.info( + `๐ŸŽฏ Processing job ${i + 1}/${jobs.length} (spanId: ${job.spanId})` + ); + const evalsInJob = await this.executeEvalsForJob( + logger, + job, + evalAPI, + evalMetadataMap + ); + totalEvalsExecuted += evalsInJob; + logger.info( + `โœ… Completed job ${i + 1}/${jobs.length}: ${evalsInJob} evals executed` + ); + } + + logger.info( + `โœ… Completed eval execution for session ${sessionId}: ${totalEvalsExecuted} evals executed` + ); + + // Clean up completed jobs + logger.info(`๐Ÿงน Cleaning up ${jobs.length} completed jobs...`); + for (const job of jobs) { + evalJobScheduler.removeJob(job.spanId); + } + logger.info(`โœ… Cleaned up ${jobs.length} completed jobs`); + } catch (error) { + logger.error('โŒ Error executing evals for session:', error); + } + } + + /** + * Execute evals for a specific job + */ + private async executeEvalsForJob( + logger: Logger, + job: { + spanId: string; + sessionId: string; + promptMetadata: Array<{ evals?: string[] }>; + input?: string; + output?: string; + }, + evalAPI: EvalAPI, + evalMetadataMap: Map + ): Promise { + let evalsExecuted = 0; + + logger.info( + `๐ŸŽฏ Processing job ${job.spanId} with ${job.promptMetadata.length} prompt metadata entries` + ); + + for (const promptMeta of job.promptMetadata || []) { + if (!promptMeta.evals || promptMeta.evals.length === 0) { + logger.debug('โญ๏ธ Skipping prompt metadata with no evals'); + continue; + } + + logger.info( + `๐Ÿ“ Found ${promptMeta.evals.length} evals for prompt: ${promptMeta.evals.join(', ')}` + ); + + for (const evalSlug of promptMeta.evals) { + try { + const evalId = evalMetadataMap.get(evalSlug) || evalSlug; + logger.info( + `๐Ÿš€ Running eval '${evalSlug}' (ID: ${evalId}) for session ${job.sessionId}` + ); + + const result = await evalAPI.runEval( + evalSlug, + job.input || '', + job.output || '', + job.sessionId, + job.spanId, + evalId + ); + + if (result.success) { + logger.info(`โœ… Successfully executed eval '${evalSlug}'`); + evalsExecuted++; + } else { + logger.warn( + `โš ๏ธ Eval '${evalSlug}' completed but returned error: ${result.message}` + ); + } + } catch (error) { + logger.error(`โŒ Failed to execute eval '${evalSlug}':`, error); + // Continue with other evals even if one fails + } + } + } + + logger.info( + `๐Ÿ“Š Job ${job.spanId} completed: ${evalsExecuted} evals executed` + ); + return evalsExecuted; + } } diff --git a/src/router/router.ts b/src/router/router.ts index abd9109d..9f937058 100644 --- a/src/router/router.ts +++ b/src/router/router.ts @@ -413,8 +413,14 @@ export function createRouter(config: RouterConfig): ServerRoute['handler'] { throw err; } } - ).then((r) => { - contextHandler.waitUntilAll(logger, sessionId); + ).then(async (r) => { + logger.info( + `๐Ÿ” Router calling waitUntilAll for session ${sessionId}` + ); + await contextHandler.waitUntilAll(logger, sessionId); + logger.info( + `โœ… Router completed waitUntilAll for session ${sessionId}` + ); return r; }); }); diff --git a/src/server/bun.ts b/src/server/bun.ts index 7e9869b0..9f00ad4a 100644 --- a/src/server/bun.ts +++ b/src/server/bun.ts @@ -1,5 +1,6 @@ import type { ReadableStream } from 'node:stream/web'; import { context, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; +import EvalJobScheduler from '../apis/evaljobscheduler'; import { isIdle } from '../router/context'; import type { AgentResponseData, @@ -60,6 +61,13 @@ export class BunServer implements Server { const devmode = process.env.AGENTUITY_SDK_DEV_MODE === 'true'; const { sdkVersion, logger } = this.config; + + // Make internal logger available globally for patches + (globalThis as any).__agentuityInternalLogger = logger; + + // Initialize EvalJobScheduler globally for patches + await EvalJobScheduler.getInstance(); + const internalRoutes = new InternalRoutesHandler(logger); const hostname = process.env.AGENTUITY_ENV === 'development' ? '127.0.0.1' : '0.0.0.0'; diff --git a/src/server/internal-routes.ts b/src/server/internal-routes.ts index fe1ce938..9cebe019 100644 --- a/src/server/internal-routes.ts +++ b/src/server/internal-routes.ts @@ -1,4 +1,3 @@ -import EvalAPI from '../apis/eval'; import type { Logger } from '../logger'; import type { ServerRequest } from './types'; @@ -7,85 +6,8 @@ import type { ServerRequest } from './types'; * These routes are used internally by the SDK and should not be exposed to users */ export class InternalRoutesHandler { - private logger: Logger; - - constructor(logger: Logger) { - this.logger = logger; - } - - /** - * Handles internal eval routes: /_agentuity/eval/:evalId (POST) - */ - async handleEval(req: ServerRequest): Promise { - if (req.method !== 'POST') { - return new Response('Method Not Allowed', { status: 405 }); - } - - try { - // Parse the eval ID from the URL path - // Expected format: /_agentuity/eval/{evalId} - const pathParts = req.url.split('/'); - const evalIdIndex = pathParts.indexOf('eval'); - if (evalIdIndex === -1 || evalIdIndex + 1 >= pathParts.length) { - return new Response('Invalid eval ID in path', { status: 400 }); - } - const evalId = pathParts[evalIdIndex + 1]; - - // Parse request body - const body = await this.parseRequestBody(req); - if (!body) { - return new Response('Invalid request body', { status: 400 }); - } - - // Type assertion for the body - const evalBody = body as { - input: string; - output: string; - sessionId: string; - spanId: string; - evalName: string; - }; - - // Validate required fields - if ( - !evalBody.input || - !evalBody.output || - !evalBody.sessionId || - !evalBody.spanId || - !evalBody.evalName - ) { - return new Response('Missing required fields', { status: 400 }); - } - - const evalAPI = new EvalAPI(); - - // Run the eval - const result = await evalAPI.runEval( - evalBody.evalName, // evalName (slug) - evalBody.input, - evalBody.output, - evalBody.sessionId, - evalBody.spanId, - evalId // evalId from URL path - ); - - return new Response(JSON.stringify(result), { - status: 200, - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - }, - }); - } catch (error) { - this.logger.error('Internal eval route error:', error); - return new Response(JSON.stringify({ error: (error as Error).message }), { - status: 500, - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - }, - }); - } + constructor(_logger: Logger) { + // Logger parameter kept for future use } /** @@ -103,48 +25,9 @@ export class InternalRoutesHandler { return null; } - // Handle eval routes - if (req.url.startsWith('/_agentuity/eval/')) { - return this.handleEval(req); - } - // Add more internal routes here as needed // e.g., /_agentuity/analytics, /_agentuity/metrics, etc. return new Response('Internal route not found', { status: 404 }); } - - /** - * Parses request body from different request types - */ - private async parseRequestBody(req: ServerRequest): Promise { - if (req.body) { - // Handle ReadableStream body - if (req.body instanceof ReadableStream) { - const reader = req.body.getReader(); - const chunks: Uint8Array[] = []; - let done = false; - - while (!done) { - const { value, done: readerDone } = await reader.read(); - done = readerDone; - if (value) { - chunks.push(value); - } - } - - const bodyText = new TextDecoder().decode( - new Uint8Array( - chunks.reduce((acc, chunk) => { - acc.push(...chunk); - return acc; - }, [] as number[]) - ) - ); - return JSON.parse(bodyText); - } - } - - return null; - } } diff --git a/src/server/node.ts b/src/server/node.ts index f3c0d3da..a493c2a8 100644 --- a/src/server/node.ts +++ b/src/server/node.ts @@ -5,6 +5,7 @@ import { import { Readable } from 'node:stream'; import type { ReadableStream } from 'node:stream/web'; import { context, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; +import EvalJobScheduler from '../apis/evaljobscheduler'; import type { Logger } from '../logger'; import { isIdle } from '../router/context'; import type { AgentResponseData, AgentWelcomeResult } from '../types'; @@ -105,6 +106,10 @@ export class NodeServer implements Server { async start(): Promise { const sdkVersion = this.sdkVersion; const devmode = process.env.AGENTUITY_SDK_DEV_MODE === 'true'; + + // Initialize EvalJobScheduler globally for patches + await EvalJobScheduler.getInstance(); + const internalRoutes = new InternalRoutesHandler(this.logger); this.server = createHttpServer(async (req, res) => { if (req.method === 'GET' && req.url === '/_health') { From 084b3ae4b7f28f8188a41cc269ec0a0e32b033a4 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Tue, 21 Oct 2025 12:32:36 -0400 Subject: [PATCH 20/26] evals are running again --- src/apis/eval.ts | 100 ++++++++++++++++++++++++++++++++++++++---- src/router/context.ts | 6 +-- 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/src/apis/eval.ts b/src/apis/eval.ts index b426d37e..79c13c18 100644 --- a/src/apis/eval.ts +++ b/src/apis/eval.ts @@ -104,6 +104,59 @@ export default class EvalAPI { ); } + /** + * Load eval function and metadata by name/slug + * Scans through all eval files to find the one with matching slug + */ + async loadEvalByName(evalName: string): Promise<{ + evalFn: EvalFunction; + metadata: { id: string; slug: string; name: string; description: string }; + }> { + internal.debug(`Loading eval by name: ${evalName}`); + + try { + // Get all files in the evals directory + const files = fs.readdirSync(this.evalsDir); + + for (const file of files) { + // Skip index files and non-eval files + if (file === 'index.ts' || file === 'index.js') { + continue; + } + + // Check file extension based on bundled state + const expectedExt = this.isBundled ? '.js' : '.ts'; + if (!file.endsWith(expectedExt)) { + continue; + } + + const filePath = path.join(this.evalsDir, file); + + try { + // Convert to file URL for proper ESM import + const fileUrl = pathToFileURL(filePath).href; + const module = await import(fileUrl); + + // Check if this module has the matching slug + if (module.metadata && module.metadata.slug === evalName) { + internal.debug(`Found eval with slug ${evalName} in file ${file}`); + return { + evalFn: module.default, + metadata: module.metadata, + }; + } + } catch (error) { + // Skip files that can't be imported (might not be eval files) + internal.debug(`Skipping file ${file} due to import error: ${error}`); + } + } + + throw new Error(`No eval found with slug: ${evalName}`); + } catch (error) { + throw new Error(`Failed to load eval by name ${evalName}: ${error}`); + } + } + /** * Load eval function and metadata by ID * Scans through all eval files to find the one with matching ID @@ -166,7 +219,6 @@ export default class EvalAPI { output: string, sessionId: string, spanId: string, - evalId: string, promptHash?: string ): Promise { internal.debug(`Running eval ${evalName} for session ${sessionId}`); @@ -186,8 +238,12 @@ export default class EvalAPI { internal.debug('loading eval function'); try { - // Try to load by ID first, fallback to name - const { evalFn } = await this.loadEvalById(evalId); + // Load eval by name/slug + const { evalFn, metadata } = await this.loadEvalByName(evalName); + + if (!metadata?.id) { + throw new Error('Eval metadata not found'); + } const response: EvalResponse = { pass: ( @@ -206,7 +262,7 @@ export default class EvalAPI { ...meta, }, }, - evalId, + evalId: metadata.id, promptHash, }; }, @@ -226,7 +282,7 @@ export default class EvalAPI { ...meta, }, }, - evalId, + evalId: metadata.id, promptHash, }; }, @@ -239,19 +295,45 @@ export default class EvalAPI { throw new Error('Eval function did not call res.pass() or res.score()'); } + const r = createEvalRunRequest as CreateEvalRunRequest; + + // Just return the result directly - no need to POST to API since we're executing locally + internal.info(`โœ… Eval '${evalName}' completed successfully: %j`, { + resultType: r.result.success ? 'success' : 'error', + passed: 'passed' in r.result ? r.result.passed : undefined, + score: 'score' in r.result ? r.result.score : undefined, + metadata: r.result.metadata, + }); const resp = await POST( - `/_agentuity/eval/${evalId}/runs`, + '/evalrun/finish', JSON.stringify(createEvalRunRequest), { 'Content-Type': 'application/json', } ); - if (!resp.json?.success) { - throw new Error('Failed to create eval run'); + if (!resp.status) { + internal.info('Failed to update the database with the eval run', { + status: resp.status, + evalName, + evalId: r.evalId, + sessionId: r.sessionId, + }); + } else { + internal.info('Eval run updated in the database', { + status: resp.status, + evalName, + evalId: r.evalId, + sessionId: r.sessionId, + }); } - return resp.json; + return { + success: true, + data: { + id: `local-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, + }, + }; } catch (error) { // Return error result if eval function throws return { diff --git a/src/router/context.ts b/src/router/context.ts index 4b4d2d47..fe7e77ad 100644 --- a/src/router/context.ts +++ b/src/router/context.ts @@ -203,9 +203,8 @@ export default class AgentContextWaitUntilHandler { for (const evalSlug of promptMeta.evals) { try { - const evalId = evalMetadataMap.get(evalSlug) || evalSlug; logger.info( - `๐Ÿš€ Running eval '${evalSlug}' (ID: ${evalId}) for session ${job.sessionId}` + `๐Ÿš€ Running eval '${evalSlug}' for session ${job.sessionId}` ); const result = await evalAPI.runEval( @@ -213,8 +212,7 @@ export default class AgentContextWaitUntilHandler { job.input || '', job.output || '', job.sessionId, - job.spanId, - evalId + job.spanId ); if (result.success) { From cba15da9d36cafa143d200085dc3b73718e6ebbe Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Tue, 21 Oct 2025 16:07:08 -0400 Subject: [PATCH 21/26] added rest of the important shit --- src/index.ts | 1 + src/router/context.ts | 9 ++++-- src/utils/hash.ts | 29 ++++++++++++++++++ src/utils/promptMetadata.ts | 12 ++------ test/utils/hash.test.ts | 61 +++++++++++++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 src/utils/hash.ts create mode 100644 test/utils/hash.test.ts diff --git a/src/index.ts b/src/index.ts index 8206c275..0915cede 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,7 @@ export * from './apis/eval'; export * from './logger'; export * from './server'; export * from './types'; +export { hash, hashSync } from './utils/hash'; export * from './utils/interpolate'; export { type PromptAttributes, diff --git a/src/router/context.ts b/src/router/context.ts index fe7e77ad..68d042b4 100644 --- a/src/router/context.ts +++ b/src/router/context.ts @@ -8,6 +8,7 @@ import EvalAPI from '../apis/eval'; import EvalJobScheduler from '../apis/evaljobscheduler'; import { markSessionCompleted } from '../apis/session'; import type { Logger } from '../logger'; +import type { PromptAttributes } from '../utils/promptMetadata'; let running = 0; export function isIdle(): boolean { @@ -178,7 +179,7 @@ export default class AgentContextWaitUntilHandler { job: { spanId: string; sessionId: string; - promptMetadata: Array<{ evals?: string[] }>; + promptMetadata: PromptAttributes[]; input?: string; output?: string; }, @@ -207,12 +208,16 @@ export default class AgentContextWaitUntilHandler { `๐Ÿš€ Running eval '${evalSlug}' for session ${job.sessionId}` ); + logger.info(`๐Ÿ”‘ Template hash: ${promptMeta.templateHash}`); + logger.info(`๐Ÿ”‘ Compiled hash: ${promptMeta.compiledHash}`); + const result = await evalAPI.runEval( evalSlug, job.input || '', job.output || '', job.sessionId, - job.spanId + job.spanId, + promptMeta.templateHash ); if (result.success) { diff --git a/src/utils/hash.ts b/src/utils/hash.ts new file mode 100644 index 00000000..a227d66f --- /dev/null +++ b/src/utils/hash.ts @@ -0,0 +1,29 @@ +import crypto from 'crypto'; + +/** + * Convert an ArrayBuffer to a hexadecimal string + */ +function arrayBufferToHex(arrayBuffer: ArrayBuffer): string { + const array = Array.from(new Uint8Array(arrayBuffer)); + const hex = array.map((byte) => byte.toString(16).padStart(2, '0')).join(''); + return hex; +} + +/** + * Hash a string using SHA-256 and return the hexadecimal representation (async) + */ +export async function hash(value: string): Promise { + const ctBuffer = await crypto.subtle.digest( + 'SHA-256', + new TextEncoder().encode(value) + ); + return arrayBufferToHex(ctBuffer); +} + +/** + * Hash a string using SHA-256 and return the hexadecimal representation (sync) + * Uses Node.js crypto for synchronous operation + */ +export function hashSync(value: string): string { + return crypto.createHash('sha256').update(value).digest('hex'); +} diff --git a/src/utils/promptMetadata.ts b/src/utils/promptMetadata.ts index 22e8dffc..c612bc40 100644 --- a/src/utils/promptMetadata.ts +++ b/src/utils/promptMetadata.ts @@ -1,6 +1,7 @@ import crypto from 'crypto'; import PatchPortal from '../apis/patchportal.js'; import { internal } from '../logger/internal'; +import { hashSync } from './hash.js'; export interface PromptAttributesParams { slug: string; @@ -34,17 +35,10 @@ export async function processPromptMetadata( internal.debug('โœ… PatchPortal instance obtained'); // Generate hash - const templateHash = crypto - .createHash('sha256') - .update(attributes.template) - .digest('hex'); - + const templateHash = hashSync(attributes.template); internal.debug('๐Ÿ”‘ Template hash:', templateHash); - const compiledHash = crypto - .createHash('sha256') - .update(attributes.compiled) - .digest('hex'); + const compiledHash = hashSync(attributes.compiled); internal.debug('๐Ÿ”‘ Compiled hash:', compiledHash); diff --git a/test/utils/hash.test.ts b/test/utils/hash.test.ts new file mode 100644 index 00000000..aca7f640 --- /dev/null +++ b/test/utils/hash.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'bun:test'; +import { hash, hashSync } from '../../src/utils/hash'; + +describe('Hash Functions', () => { + it('should produce the same hash for both async and sync functions', async () => { + const testCases = [ + 'Hello, World!', + '', + 'This is a longer string with special characters: !@#$%^&*()', + 'Unicode test: ๐Ÿš€ ๐ŸŒŸ ๐ŸŽ‰', + 'Multiple\nlines\nwith\ttabs', + 'Very long string '.repeat(100), + ]; + + for (const testString of testCases) { + const asyncHash = await hash(testString); + const syncHash = hashSync(testString); + + expect(asyncHash).toBe(syncHash); + expect(asyncHash).toMatch(/^[a-f0-9]{64}$/); // SHA-256 produces 64 hex characters + expect(syncHash).toMatch(/^[a-f0-9]{64}$/); + } + }); + + it('should produce consistent hashes for the same input', async () => { + const testString = 'Consistent hash test'; + + const hash1 = await hash(testString); + const hash2 = await hash(testString); + const syncHash1 = hashSync(testString); + const syncHash2 = hashSync(testString); + + expect(hash1).toBe(hash2); + expect(syncHash1).toBe(syncHash2); + expect(hash1).toBe(syncHash1); + }); + + it('should produce different hashes for different inputs', async () => { + const string1 = 'Hello'; + const string2 = 'World'; + + const hash1 = await hash(string1); + const hash2 = await hash(string2); + const syncHash1 = hashSync(string1); + const syncHash2 = hashSync(string2); + + expect(hash1).not.toBe(hash2); + expect(syncHash1).not.toBe(syncHash2); + expect(hash1).toBe(syncHash1); + expect(hash2).toBe(syncHash2); + }); + + it('should handle empty string', async () => { + const emptyString = ''; + const asyncHash = await hash(emptyString); + const syncHash = hashSync(emptyString); + + expect(asyncHash).toBe(syncHash); + expect(asyncHash).toMatch(/^[a-f0-9]{64}$/); + }); +}); From 232fb4b25b83cc8ce0d1bf5a70630424d93f7037 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Wed, 22 Oct 2025 11:03:21 -0400 Subject: [PATCH 22/26] remove dev change --- package.json | 1 - src/server/bun.ts | 3 --- 2 files changed, 4 deletions(-) diff --git a/package.json b/package.json index 6ca3061d..53477bf0 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,6 @@ "esbuild": ">=0.25.0" }, "dependencies": { - "@clickhouse/client": "^1.0.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/api-logs": "^0.57.2", "@opentelemetry/auto-instrumentations-node": "^0.56.1", diff --git a/src/server/bun.ts b/src/server/bun.ts index 9f00ad4a..f500dc8b 100644 --- a/src/server/bun.ts +++ b/src/server/bun.ts @@ -62,9 +62,6 @@ export class BunServer implements Server { const devmode = process.env.AGENTUITY_SDK_DEV_MODE === 'true'; const { sdkVersion, logger } = this.config; - // Make internal logger available globally for patches - (globalThis as any).__agentuityInternalLogger = logger; - // Initialize EvalJobScheduler globally for patches await EvalJobScheduler.getInstance(); From 1fbb376bbdb337c4f79e50b4d9ba41e4bfe66284 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Wed, 22 Oct 2025 11:38:42 -0400 Subject: [PATCH 23/26] remove route changes --- package-lock.json | 22 ++-------------------- src/server/bun.ts | 17 ----------------- src/server/internal-routes.ts | 33 --------------------------------- src/server/node.ts | 24 ------------------------ 4 files changed, 2 insertions(+), 94 deletions(-) delete mode 100644 src/server/internal-routes.ts diff --git a/package-lock.json b/package-lock.json index 4ac1367c..8c4f2bbf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,6 @@ "version": "0.0.156", "license": "Apache-2.0", "dependencies": { - "@clickhouse/client": "^1.0.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/api-logs": "^0.57.2", "@opentelemetry/auto-instrumentations-node": "^0.56.1", @@ -1109,24 +1108,6 @@ "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", "license": "MIT" }, - "node_modules/@clickhouse/client": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@clickhouse/client/-/client-1.12.1.tgz", - "integrity": "sha512-7ORY85rphRazqHzImNXMrh4vsaPrpetFoTWpZYueCO2bbO6PXYDXp/GQ4DgxnGIqbWB/Di1Ai+Xuwq2o7DJ36A==", - "license": "Apache-2.0", - "dependencies": { - "@clickhouse/client-common": "1.12.1" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/@clickhouse/client-common": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@clickhouse/client-common/-/client-common-1.12.1.tgz", - "integrity": "sha512-ccw1N6hB4+MyaAHIaWBwGZ6O2GgMlO99FlMj0B0UEGfjxM9v5dYVYql6FpP19rMwrVAroYs/IgX2vyZEBvzQLg==", - "license": "Apache-2.0" - }, "node_modules/@esbuild/aix-ppc64": { "version": "0.25.9", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.9.tgz", @@ -6843,6 +6824,7 @@ "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -9416,4 +9398,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/server/bun.ts b/src/server/bun.ts index f500dc8b..b2aefd5f 100644 --- a/src/server/bun.ts +++ b/src/server/bun.ts @@ -7,7 +7,6 @@ import type { AgentWelcomeResult, ReadableDataType, } from '../types'; -import { InternalRoutesHandler } from './internal-routes'; import { extractTraceContextFromBunRequest, injectTraceContextToHeaders, @@ -65,7 +64,6 @@ export class BunServer implements Server { // Initialize EvalJobScheduler globally for patches await EvalJobScheduler.getInstance(); - const internalRoutes = new InternalRoutesHandler(logger); const hostname = process.env.AGENTUITY_ENV === 'development' ? '127.0.0.1' : '0.0.0.0'; @@ -172,21 +170,6 @@ export class BunServer implements Server { const url = new URL(req.url); - // Handle internal routes first - if (url.pathname.startsWith('/_agentuity/')) { - const internalResponse = await internalRoutes.handleInternalRoute({ - method, - url: url.pathname, - headers: req.headers.toJSON(), - body: req.body as unknown as ReadableStream, - request: getRequestFromHeaders(req.headers.toJSON(), ''), - setTimeout: (_val: number) => void 0, - }); - if (internalResponse) { - return internalResponse; - } - } - // Extract trace context from headers const extractedContext = extractTraceContextFromBunRequest(req); diff --git a/src/server/internal-routes.ts b/src/server/internal-routes.ts deleted file mode 100644 index 9cebe019..00000000 --- a/src/server/internal-routes.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { Logger } from '../logger'; -import type { ServerRequest } from './types'; - -/** - * Handles internal /_agentuity routes - * These routes are used internally by the SDK and should not be exposed to users - */ -export class InternalRoutesHandler { - constructor(_logger: Logger) { - // Logger parameter kept for future use - } - - /** - * Checks if a request is for an internal route - */ - isInternalRoute(url: string): boolean { - return url.startsWith('/_agentuity/'); - } - - /** - * Routes internal requests to appropriate handlers - */ - async handleInternalRoute(req: ServerRequest): Promise { - if (!this.isInternalRoute(req.url)) { - return null; - } - - // Add more internal routes here as needed - // e.g., /_agentuity/analytics, /_agentuity/metrics, etc. - - return new Response('Internal route not found', { status: 404 }); - } -} diff --git a/src/server/node.ts b/src/server/node.ts index a493c2a8..705e41b2 100644 --- a/src/server/node.ts +++ b/src/server/node.ts @@ -9,7 +9,6 @@ import EvalJobScheduler from '../apis/evaljobscheduler'; import type { Logger } from '../logger'; import { isIdle } from '../router/context'; import type { AgentResponseData, AgentWelcomeResult } from '../types'; -import { InternalRoutesHandler } from './internal-routes'; import { extractTraceContextFromNodeRequest, injectTraceContextToHeaders, @@ -110,7 +109,6 @@ export class NodeServer implements Server { // Initialize EvalJobScheduler globally for patches await EvalJobScheduler.getInstance(); - const internalRoutes = new InternalRoutesHandler(this.logger); this.server = createHttpServer(async (req, res) => { if (req.method === 'GET' && req.url === '/_health') { res.writeHead(200, { @@ -140,28 +138,6 @@ export class NodeServer implements Server { return; } - // Handle internal routes first - if (req.url?.startsWith('/_agentuity/')) { - const body = await this.getBufferAsStream(req); - const internalResponse = await internalRoutes.handleInternalRoute({ - method: req.method || 'GET', - url: req.url, - headers: this.getHeaders(req), - body, - request: getRequestFromHeaders(this.getHeaders(req), ''), - setTimeout: (_val: number) => void 0, - }); - if (internalResponse) { - const responseBody = await internalResponse.text(); - res.writeHead(internalResponse.status, { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - }); - res.end(responseBody); - return; - } - } - if (req.method === 'GET' && req.url === '/welcome') { const result: Record = {}; for (const route of this.routes) { From 0dd2845ffb3ce219cfd52b205eab78d1d7aed842 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Wed, 22 Oct 2025 11:52:49 -0400 Subject: [PATCH 24/26] code review --- src/apis/api.ts | 1 - src/apis/eval.ts | 20 ++++++------- src/apis/evaljobscheduler.ts | 15 ++++++---- src/router/context.ts | 54 +++++++++++++++++++----------------- 4 files changed, 47 insertions(+), 43 deletions(-) diff --git a/src/apis/api.ts b/src/apis/api.ts index 1f9798b3..3991f111 100644 --- a/src/apis/api.ts +++ b/src/apis/api.ts @@ -176,7 +176,6 @@ export async function send( init.keepalive = true; } - console.log('BOBBY!! url', url); const resp = await apiFetch(url, init); let json: K | null = null; switch (resp.status) { diff --git a/src/apis/eval.ts b/src/apis/eval.ts index 79c13c18..5613392e 100644 --- a/src/apis/eval.ts +++ b/src/apis/eval.ts @@ -297,8 +297,7 @@ export default class EvalAPI { const r = createEvalRunRequest as CreateEvalRunRequest; - // Just return the result directly - no need to POST to API since we're executing locally - internal.info(`โœ… Eval '${evalName}' completed successfully: %j`, { + internal.debug(`โœ… Eval '${evalName}' completed successfully: %j`, { resultType: r.result.success ? 'success' : 'error', passed: 'passed' in r.result ? r.result.passed : undefined, score: 'score' in r.result ? r.result.score : undefined, @@ -309,18 +308,19 @@ export default class EvalAPI { JSON.stringify(createEvalRunRequest), { 'Content-Type': 'application/json', - } + }, + 'eval' ); if (!resp.status) { - internal.info('Failed to update the database with the eval run', { + internal.debug('Failed to update the database with the eval run', { status: resp.status, evalName, evalId: r.evalId, sessionId: r.sessionId, }); } else { - internal.info('Eval run updated in the database', { + internal.debug('Eval run updated in the database', { status: resp.status, evalName, evalId: r.evalId, @@ -348,11 +348,11 @@ export default class EvalAPI { * Scans through all eval files to find metadata and build mapping */ async loadEvalMetadataMap(): Promise> { - internal.info(`๐Ÿ” Loading eval metadata map from: ${this.evalsDir}`); + internal.debug(`๐Ÿ” Loading eval metadata map from: ${this.evalsDir}`); // Check if evals directory exists if (!fs.existsSync(this.evalsDir)) { - internal.info(`๐Ÿ“ Evals directory not found: ${this.evalsDir}`); + internal.debug(`๐Ÿ“ Evals directory not found: ${this.evalsDir}`); return new Map(); } @@ -360,7 +360,7 @@ export default class EvalAPI { const slugToIDMap = new Map(); let processedFiles = 0; - internal.info(`๐Ÿ“‚ Scanning ${files.length} files in evals directory`); + internal.debug(`๐Ÿ“‚ Scanning ${files.length} files in evals directory`); for (const file of files) { const ext = path.extname(file); @@ -382,7 +382,7 @@ export default class EvalAPI { if (metadata && metadata.slug && metadata.id) { slugToIDMap.set(metadata.slug, metadata.id); - internal.info( + internal.debug( `โœ… Mapped eval slug '${metadata.slug}' to ID '${metadata.id}' from ${file}` ); } else { @@ -393,7 +393,7 @@ export default class EvalAPI { } } - internal.info( + internal.debug( `๐Ÿ“š Loaded ${slugToIDMap.size} eval mappings from ${processedFiles} files` ); return slugToIDMap; diff --git a/src/apis/evaljobscheduler.ts b/src/apis/evaljobscheduler.ts index 5602573d..2fedb322 100644 --- a/src/apis/evaljobscheduler.ts +++ b/src/apis/evaljobscheduler.ts @@ -85,7 +85,7 @@ export default class EvalJobScheduler { (count, meta) => count + (meta.evals?.length || 0), 0 ); - internal.info( + internal.debug( `๐Ÿ“ฆ Creating eval job ${spanId} for session ${sessionId} with ${totalEvals} evals` ); @@ -117,23 +117,26 @@ export default class EvalJobScheduler { * Get jobs with optional filtering */ public getJobs(filter?: JobFilter): PendingEvalJob[] { - internal.info('๐Ÿ” EvalJobScheduler.getJobs() called with filter:', filter); - internal.info('๐Ÿ“Š Total pending jobs in scheduler:', this.pendingJobs.size); + internal.debug('๐Ÿ” EvalJobScheduler.getJobs() called with filter:', filter); + internal.debug( + '๐Ÿ“Š Total pending jobs in scheduler:', + this.pendingJobs.size + ); let jobs = Array.from(this.pendingJobs.values()); if (filter?.sessionId) { jobs = jobs.filter((job) => job.sessionId === filter.sessionId); - internal.info( + internal.debug( `๐Ÿ” Filtered jobs for session ${filter.sessionId}:`, jobs.length ); } - internal.info('๐Ÿ“‹ Returning jobs:', jobs.length); + internal.debug('๐Ÿ“‹ Returning jobs:', jobs.length); if (jobs.length > 0) { jobs.forEach((job, index) => { - internal.info(`๐Ÿ“ฆ Job ${index + 1}:`, { + internal.debug(`๐Ÿ“ฆ Job ${index + 1}:`, { spanId: job.spanId, sessionId: job.sessionId, promptMetadataCount: job.promptMetadata?.length || 0, diff --git a/src/router/context.ts b/src/router/context.ts index 68d042b4..3b4310ed 100644 --- a/src/router/context.ts +++ b/src/router/context.ts @@ -8,6 +8,7 @@ import EvalAPI from '../apis/eval'; import EvalJobScheduler from '../apis/evaljobscheduler'; import { markSessionCompleted } from '../apis/session'; import type { Logger } from '../logger'; +import { internal } from '../logger/internal'; import type { PromptAttributes } from '../utils/promptMetadata'; let running = 0; @@ -71,7 +72,7 @@ export default class AgentContextWaitUntilHandler { } public async waitUntilAll(logger: Logger, sessionId: string): Promise { - logger.info(`๐Ÿ” waitUntilAll() called for session ${sessionId}`); + internal.debug(`๐Ÿ” waitUntilAll() called for session ${sessionId}`); if (this.hasCalledWaitUntilAll) { throw new Error('waitUntilAll can only be called once per instance'); @@ -79,24 +80,23 @@ export default class AgentContextWaitUntilHandler { this.hasCalledWaitUntilAll = true; if (this.promises.length === 0) { - logger.info('๐Ÿ“ญ No promises to wait for, executing evals directly'); - // Execute evals even if no promises + internal.debug('No promises to wait for, executing evals directly'); await this.executeEvalsForSession(logger, sessionId); return; } - logger.info( + internal.debug( `โณ Waiting for ${this.promises.length} promises to complete...` ); try { // Promises are already executing, just wait for them to complete await Promise.all(this.promises); const duration = Date.now() - (this.started as number); - logger.info('โœ… All promises completed, marking session completed'); + internal.debug('โœ… All promises completed, marking session completed'); await markSessionCompleted(sessionId, duration); // Execute evals after session completion - logger.info('๐Ÿš€ Starting eval execution after session completion'); + internal.debug('๐Ÿš€ Starting eval execution after session completion'); await this.executeEvalsForSession(logger, sessionId); } catch (ex) { logger.error('error sending session completed', ex); @@ -114,34 +114,36 @@ export default class AgentContextWaitUntilHandler { sessionId: string ): Promise { try { - logger.info(`๐Ÿ” Starting eval execution for session ${sessionId}`); + internal.debug(`๐Ÿ” Starting eval execution for session ${sessionId}`); // Get pending eval jobs for this session - logger.info('๐Ÿ” Getting EvalJobScheduler instance...'); + internal.debug('๐Ÿ” Getting EvalJobScheduler instance...'); const evalJobScheduler = await EvalJobScheduler.getInstance(); - logger.info('โœ… EvalJobScheduler instance obtained'); + internal.debug('โœ… EvalJobScheduler instance obtained'); - logger.info(`๐Ÿ” Querying jobs for session ${sessionId}...`); + internal.debug(`๐Ÿ” Querying jobs for session ${sessionId}...`); const jobs = evalJobScheduler.getJobs({ sessionId }); if (jobs.length === 0) { - logger.info(`๐Ÿ“ญ No eval jobs found for session ${sessionId}`); + internal.debug(`๐Ÿ“ญ No eval jobs found for session ${sessionId}`); return; } - logger.info(`๐Ÿ“‹ Found ${jobs.length} eval jobs for session ${sessionId}`); + internal.debug( + `๐Ÿ“‹ Found ${jobs.length} eval jobs for session ${sessionId}` + ); // Load eval metadata map - logger.info('๐Ÿ”ง Loading eval metadata map...'); + internal.debug('๐Ÿ”ง Loading eval metadata map...'); const evalAPI = new EvalAPI(); const evalMetadataMap = await evalAPI.loadEvalMetadataMap(); - logger.info(`๐Ÿ“š Loaded ${evalMetadataMap.size} eval mappings`); + internal.debug(`๐Ÿ“š Loaded ${evalMetadataMap.size} eval mappings`); // Execute evals for each job let totalEvalsExecuted = 0; for (let i = 0; i < jobs.length; i++) { const job = jobs[i]; - logger.info( + internal.debug( `๐ŸŽฏ Processing job ${i + 1}/${jobs.length} (spanId: ${job.spanId})` ); const evalsInJob = await this.executeEvalsForJob( @@ -151,21 +153,21 @@ export default class AgentContextWaitUntilHandler { evalMetadataMap ); totalEvalsExecuted += evalsInJob; - logger.info( + internal.debug( `โœ… Completed job ${i + 1}/${jobs.length}: ${evalsInJob} evals executed` ); } - logger.info( + internal.debug( `โœ… Completed eval execution for session ${sessionId}: ${totalEvalsExecuted} evals executed` ); // Clean up completed jobs - logger.info(`๐Ÿงน Cleaning up ${jobs.length} completed jobs...`); + internal.debug(`๐Ÿงน Cleaning up ${jobs.length} completed jobs...`); for (const job of jobs) { evalJobScheduler.removeJob(job.spanId); } - logger.info(`โœ… Cleaned up ${jobs.length} completed jobs`); + internal.debug(`โœ… Cleaned up ${jobs.length} completed jobs`); } catch (error) { logger.error('โŒ Error executing evals for session:', error); } @@ -188,7 +190,7 @@ export default class AgentContextWaitUntilHandler { ): Promise { let evalsExecuted = 0; - logger.info( + internal.debug( `๐ŸŽฏ Processing job ${job.spanId} with ${job.promptMetadata.length} prompt metadata entries` ); @@ -198,18 +200,18 @@ export default class AgentContextWaitUntilHandler { continue; } - logger.info( + internal.debug( `๐Ÿ“ Found ${promptMeta.evals.length} evals for prompt: ${promptMeta.evals.join(', ')}` ); for (const evalSlug of promptMeta.evals) { try { - logger.info( + internal.debug( `๐Ÿš€ Running eval '${evalSlug}' for session ${job.sessionId}` ); - logger.info(`๐Ÿ”‘ Template hash: ${promptMeta.templateHash}`); - logger.info(`๐Ÿ”‘ Compiled hash: ${promptMeta.compiledHash}`); + internal.debug(`๐Ÿ”‘ Template hash: ${promptMeta.templateHash}`); + internal.debug(`๐Ÿ”‘ Compiled hash: ${promptMeta.compiledHash}`); const result = await evalAPI.runEval( evalSlug, @@ -221,7 +223,7 @@ export default class AgentContextWaitUntilHandler { ); if (result.success) { - logger.info(`โœ… Successfully executed eval '${evalSlug}'`); + internal.debug(`โœ… Successfully executed eval '${evalSlug}'`); evalsExecuted++; } else { logger.warn( @@ -235,7 +237,7 @@ export default class AgentContextWaitUntilHandler { } } - logger.info( + internal.debug( `๐Ÿ“Š Job ${job.spanId} completed: ${evalsExecuted} evals executed` ); return evalsExecuted; From 55860f03335aeaf87d5904639f3c9c105c7ad756 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Wed, 22 Oct 2025 12:04:48 -0400 Subject: [PATCH 25/26] code review --- src/apis/eval.ts | 4 +++- src/apis/patchportal.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/apis/eval.ts b/src/apis/eval.ts index 5613392e..106eb751 100644 --- a/src/apis/eval.ts +++ b/src/apis/eval.ts @@ -309,6 +309,8 @@ export default class EvalAPI { { 'Content-Type': 'application/json', }, + undefined, + undefined, 'eval' ); @@ -331,7 +333,7 @@ export default class EvalAPI { return { success: true, data: { - id: `local-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, + id: `local-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`, }, }; } catch (error) { diff --git a/src/apis/patchportal.ts b/src/apis/patchportal.ts index 6010b356..31078202 100644 --- a/src/apis/patchportal.ts +++ b/src/apis/patchportal.ts @@ -14,7 +14,7 @@ export default class PatchPortal { private constructor() { // Private constructor to prevent direct instantiation - this.instanceId = `PatchPortal-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + this.instanceId = `PatchPortal-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; internal.debug( '๐Ÿ—๏ธ PatchPortal constructor called, instanceId:', this.instanceId From 3c4e1d874389ac75e581732376cd22df95927052 Mon Sep 17 00:00:00 2001 From: Bobby Christopher Date: Wed, 22 Oct 2025 14:22:03 -0400 Subject: [PATCH 26/26] added the version bump --- CHANGELOG.md | 6 ++++++ package-lock.json | 4 ++-- package.json | 2 +- src/router/router.ts | 4 ++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c08ec028..344ecf74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # @agentuity/sdk Changelog +## 0.0.157 + +### Patch Changes + +- Add support for eval running + ## [0.0.156] - 2025-10-15 ### Added diff --git a/package-lock.json b/package-lock.json index 8c4f2bbf..cc09753f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@agentuity/sdk", - "version": "0.0.156", + "version": "0.0.157", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@agentuity/sdk", - "version": "0.0.156", + "version": "0.0.157", "license": "Apache-2.0", "dependencies": { "@opentelemetry/api": "^1.9.0", diff --git a/package.json b/package.json index 53477bf0..00367907 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agentuity/sdk", - "version": "0.0.156", + "version": "0.0.157", "description": "The Agentuity SDK for NodeJS and Bun", "license": "Apache-2.0", "public": true, diff --git a/src/router/router.ts b/src/router/router.ts index 9f937058..12855225 100644 --- a/src/router/router.ts +++ b/src/router/router.ts @@ -414,11 +414,11 @@ export function createRouter(config: RouterConfig): ServerRoute['handler'] { } } ).then(async (r) => { - logger.info( + internal.info( `๐Ÿ” Router calling waitUntilAll for session ${sessionId}` ); await contextHandler.waitUntilAll(logger, sessionId); - logger.info( + internal.info( `โœ… Router completed waitUntilAll for session ${sessionId}` ); return r;