From d6b08e46912117d119751c6cbdbe88e38449edd6 Mon Sep 17 00:00:00 2001 From: Vsevolod Avramov Date: Thu, 9 Apr 2026 14:40:57 +0300 Subject: [PATCH 1/3] Add structured agent profiles --- drizzle/0004_agent_profiles.sql | 21 +++ drizzle/meta/_journal.json | 7 + src/agent-profile.ts | 247 ++++++++++++++++++++++++++++++++ src/config.ts | 13 ++ src/db/agent-profile-store.ts | 175 ++++++++++++++++++++++ src/db/schema.ts | 25 ++++ src/index.ts | 30 +++- tests/agent-profile.test.ts | 169 ++++++++++++++++++++++ 8 files changed, 685 insertions(+), 2 deletions(-) create mode 100644 drizzle/0004_agent_profiles.sql create mode 100644 src/agent-profile.ts create mode 100644 src/db/agent-profile-store.ts create mode 100644 tests/agent-profile.test.ts diff --git a/drizzle/0004_agent_profiles.sql b/drizzle/0004_agent_profiles.sql new file mode 100644 index 0000000..36327b6 --- /dev/null +++ b/drizzle/0004_agent_profiles.sql @@ -0,0 +1,21 @@ +CREATE TABLE "agent_profiles" ( + "id" uuid PRIMARY KEY NOT NULL, + "name" text NOT NULL, + "description" text, + "runtime" text NOT NULL, + "provider" text, + "model" text, + "tools" text[] DEFAULT '{}' NOT NULL, + "mode" text, + "extensions" text[] DEFAULT '{}' NOT NULL, + "extra_args" text, + "is_built_in" boolean DEFAULT false NOT NULL, + "is_active" boolean DEFAULT false NOT NULL, + "custom_command_template" text, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE INDEX "agent_profiles_active_idx" ON "agent_profiles" USING btree ("is_active"); +--> statement-breakpoint +CREATE INDEX "agent_profiles_runtime_idx" ON "agent_profiles" USING btree ("runtime"); diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index f62e791..d64ac66 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -29,6 +29,13 @@ "when": 1775570000000, "tag": "0003_breezy_config_sections", "breakpoints": true + }, + { + "idx": 4, + "version": "7", + "when": 1778600000000, + "tag": "0004_agent_profiles", + "breakpoints": true } ] } diff --git a/src/agent-profile.ts b/src/agent-profile.ts new file mode 100644 index 0000000..91a972b --- /dev/null +++ b/src/agent-profile.ts @@ -0,0 +1,247 @@ +import { randomUUID } from "node:crypto"; +import type { AppConfig } from "./config.js"; +import { shellEscape } from "./pipeline/shell.js"; + +export const AGENT_PROFILE_RUNTIMES = ["pi", "codex", "claude", "custom"] as const; +export const AGENT_PROFILE_PROVIDERS = ["openai", "openrouter", "anthropic"] as const; + +export type AgentRuntime = typeof AGENT_PROFILE_RUNTIMES[number]; +export type AgentProvider = typeof AGENT_PROFILE_PROVIDERS[number]; + +export interface AgentProfile { + id: string; + name: string; + description?: string; + runtime: AgentRuntime; + provider?: AgentProvider; + model?: string; + tools: string[]; + mode?: string; + extensions: string[]; + extraArgs?: string; + isBuiltin: boolean; + isActive: boolean; + customCommandTemplate?: string; + createdAt: string; + updatedAt: string; +} + +export interface AgentProfileInput { + name: string; + description?: string; + runtime: AgentRuntime; + provider?: AgentProvider; + model?: string; + tools?: string[]; + mode?: string; + extensions?: string[]; + extraArgs?: string; + isBuiltin?: boolean; + isActive?: boolean; + customCommandTemplate?: string; +} + +export interface AgentProfileValidationResult { + ok: boolean; + errors: string[]; +} + +export interface ProviderOption { + id: AgentProvider; + label: string; + configured: boolean; + envVar: string; +} + +const DEFAULT_PI_TOOLS = ["read", "write", "edit", "bash", "grep", "find", "ls"]; +const DEFAULT_CODEX_TOOLS = ["read", "write", "edit", "bash"]; +const DEFAULT_CLAUDE_TOOLS = ["Read", "Edit", "Write", "Bash", "Grep", "Glob"]; + +const RUNTIME_PROVIDER_SUPPORT: Record = { + pi: ["openai", "openrouter", "anthropic"], + codex: ["openai"], + claude: ["anthropic", "openrouter"], + custom: ["openai", "openrouter", "anthropic"], +}; + +function trimOrUndefined(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const trimmed = value.trim(); + return trimmed === "" ? undefined : trimmed; +} + +function uniqueStrings(values: unknown): string[] { + if (!Array.isArray(values)) return []; + return [...new Set(values.map((value) => String(value).trim()).filter(Boolean))]; +} + +export function getAvailableProviders(config: AppConfig): ProviderOption[] { + return [ + { id: "openai", label: "OpenAI", configured: Boolean(config.openaiApiKey), envVar: "OPENAI_API_KEY" }, + { id: "openrouter", label: "OpenRouter", configured: Boolean(config.openrouterApiKey), envVar: "OPENROUTER_API_KEY" }, + { id: "anthropic", label: "Anthropic", configured: Boolean(config.anthropicApiKey), envVar: "ANTHROPIC_API_KEY" }, + ]; +} + +export function isProviderConfigured(config: AppConfig, provider: AgentProvider): boolean { + return getAvailableProviders(config).some((entry) => entry.id === provider && entry.configured); +} + +export function sanitizeAgentProfileInput(input: AgentProfileInput): AgentProfileInput { + return { + name: String(input.name ?? "").trim(), + description: trimOrUndefined(input.description), + runtime: input.runtime, + provider: input.provider, + model: trimOrUndefined(input.model), + tools: uniqueStrings(input.tools), + mode: trimOrUndefined(input.mode), + extensions: uniqueStrings(input.extensions), + extraArgs: trimOrUndefined(input.extraArgs), + isBuiltin: Boolean(input.isBuiltin), + isActive: Boolean(input.isActive), + customCommandTemplate: trimOrUndefined(input.customCommandTemplate), + }; +} + +export function validateAgentProfile(input: AgentProfileInput, config: AppConfig): AgentProfileValidationResult { + const profile = sanitizeAgentProfileInput(input); + const errors: string[] = []; + + if (!AGENT_PROFILE_RUNTIMES.includes(profile.runtime)) { + errors.push("Runtime is required"); + } + if (!profile.name) { + errors.push("Name is required"); + } + + if (profile.runtime === "custom") { + if (!profile.customCommandTemplate) { + errors.push("Custom command template is required for custom profiles"); + } + return { ok: errors.length === 0, errors }; + } + + if (!profile.provider) { + errors.push("Provider is required"); + } else { + if (!RUNTIME_PROVIDER_SUPPORT[profile.runtime].includes(profile.provider)) { + errors.push(`Runtime ${profile.runtime} does not support provider ${profile.provider}`); + } + if (!isProviderConfigured(config, profile.provider)) { + errors.push(`Provider ${profile.provider} is not configured in the current environment`); + } + } + + if (!profile.model) { + errors.push("Model is required"); + } + + return { ok: errors.length === 0, errors }; +} + +export function renderAgentProfileTemplate(profile: AgentProfileInput): string { + const clean = sanitizeAgentProfileInput(profile); + if (clean.runtime === "custom") { + return clean.customCommandTemplate ?? ""; + } + + const tools = clean.tools ?? []; + const extensions = clean.extensions ?? []; + const extraArgs = clean.extraArgs ? ` ${clean.extraArgs}` : ""; + const mode = clean.mode ? ` ${clean.mode}` : ""; + + if (clean.runtime === "pi") { + const toolArg = tools.length > 0 ? ` --tools ${shellEscape(tools.join(","))}` : ""; + const extensionArg = extensions.length > 0 + ? ` ${extensions.map((extension) => `-e ${shellEscape(extension)}`).join(" ")}` + : " {{pi_extensions}}"; + return `cd {{repo_dir}} && pi -p @{{prompt_file}} --model ${shellEscape(clean.model ?? "")} --no-session --mode json${toolArg}${mode}${extensionArg}${extraArgs} {{mcp_flags}}`.trim(); + } + + if (clean.runtime === "codex") { + const modelArg = clean.model ? ` --model ${shellEscape(clean.model)}` : ""; + return `cd {{repo_dir}} && codex exec --full-auto${modelArg} "$(cat {{prompt_file}})"${extraArgs}`.trim(); + } + + const allowedTools = tools.length > 0 ? tools.join(",") : DEFAULT_CLAUDE_TOOLS.join(","); + const modelArg = clean.model ? ` --model ${shellEscape(clean.model)}` : ""; + return `cd {{repo_dir}} && claude -p "$(cat {{prompt_file}})" --allowedTools ${shellEscape(allowedTools)}${modelArg}${extraArgs}`.trim(); +} + +export function resolveProfileCommandTemplate(profile: AgentProfileInput | undefined, fallbackTemplate: string): string { + if (!profile) return fallbackTemplate; + const rendered = renderAgentProfileTemplate(profile); + return rendered || fallbackTemplate; +} + +export function buildBuiltinAgentProfiles(config: AppConfig): AgentProfileInput[] { + const profiles: AgentProfileInput[] = []; + if (config.openaiApiKey) { + profiles.push({ + name: "Pi + OpenAI", + description: "Pi agent with OpenAI-backed model selection.", + runtime: "pi", + provider: "openai", + model: "openai/gpt-4.1-mini", + tools: DEFAULT_PI_TOOLS, + isBuiltin: true, + }); + } + if (config.openrouterApiKey) { + profiles.push({ + name: "Pi + OpenRouter", + description: "Pi agent via OpenRouter for broad model access.", + runtime: "pi", + provider: "openrouter", + model: "openrouter/openai/gpt-4.1-mini", + tools: DEFAULT_PI_TOOLS, + isBuiltin: true, + }); + } + if (config.openaiApiKey) { + profiles.push({ + name: "Codex + OpenAI", + description: "Codex one-shot execution against the OpenAI Responses API.", + runtime: "codex", + provider: "openai", + model: "gpt-5.4", + tools: DEFAULT_CODEX_TOOLS, + isBuiltin: true, + }); + } + if (config.anthropicApiKey) { + profiles.push({ + name: "Claude + Anthropic", + description: "Claude CLI with Anthropic-hosted Claude models.", + runtime: "claude", + provider: "anthropic", + model: "claude-sonnet-4-6", + tools: DEFAULT_CLAUDE_TOOLS, + isBuiltin: true, + }); + } + return profiles; +} + +export function createAgentProfileRecord(input: AgentProfileInput): AgentProfile { + const clean = sanitizeAgentProfileInput(input); + const now = new Date().toISOString(); + return { + id: randomUUID(), + name: clean.name, + description: clean.description, + runtime: clean.runtime, + provider: clean.provider, + model: clean.model, + tools: clean.tools ?? [], + mode: clean.mode, + extensions: clean.extensions ?? [], + extraArgs: clean.extraArgs, + isBuiltin: Boolean(clean.isBuiltin), + isActive: Boolean(clean.isActive), + customCommandTemplate: clean.customCommandTemplate, + createdAt: now, + updatedAt: now, + }; +} diff --git a/src/config.ts b/src/config.ts index 75aac73..b61c6d3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -196,6 +196,7 @@ export interface AppConfig { gitAuthorEmail: string; agentCommandTemplate: string; + baseAgentCommandTemplate?: string; agentFollowUpTemplate?: string; validationCommand: string; lintFixCommand: string; @@ -316,6 +317,15 @@ export interface AppConfig { databaseUrl: string; encryptionKey?: string; + activeAgentProfile?: { + id: string; + name: string; + runtime: string; + provider?: string; + model?: string; + commandTemplate: string; + source: "profile" | "env"; + }; } function parseRepoMap(value?: string): Map { @@ -435,6 +445,9 @@ export function loadConfig(): AppConfig { agentCommandTemplate: parsed.AGENT_COMMAND_TEMPLATE ?? "bash scripts/dummy-agent.sh {{repo_dir}} {{prompt_file}} {{run_id}}", + baseAgentCommandTemplate: + parsed.AGENT_COMMAND_TEMPLATE ?? + "bash scripts/dummy-agent.sh {{repo_dir}} {{prompt_file}} {{run_id}}", agentFollowUpTemplate: parsed.AGENT_FOLLOW_UP_TEMPLATE?.trim() || undefined, validationCommand: parsed.VALIDATION_COMMAND ?? "", lintFixCommand: parsed.LINT_FIX_COMMAND?.trim() || "", diff --git a/src/db/agent-profile-store.ts b/src/db/agent-profile-store.ts new file mode 100644 index 0000000..7e6f616 --- /dev/null +++ b/src/db/agent-profile-store.ts @@ -0,0 +1,175 @@ +import { eq, desc, sql } from "drizzle-orm"; +import type { Database } from "./index.js"; +import { agentProfiles } from "./schema.js"; +import type { AppConfig } from "../config.js"; +import { + buildBuiltinAgentProfiles, + createAgentProfileRecord, + type AgentProfile, + type AgentProfileInput, + resolveProfileCommandTemplate, + sanitizeAgentProfileInput, + validateAgentProfile, +} from "../agent-profile.js"; + +function toRecord(row: typeof agentProfiles.$inferSelect): AgentProfile { + return { + id: row.id, + name: row.name, + description: row.description ?? undefined, + runtime: row.runtime as AgentProfile["runtime"], + provider: (row.provider ?? undefined) as AgentProfile["provider"], + model: row.model ?? undefined, + tools: row.tools ?? [], + mode: row.mode ?? undefined, + extensions: row.extensions ?? [], + extraArgs: row.extraArgs ?? undefined, + isBuiltin: row.isBuiltIn, + isActive: row.isActive, + customCommandTemplate: row.customCommandTemplate ?? undefined, + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + }; +} + +export class AgentProfileStore { + constructor( + private readonly db: Database, + private readonly config: AppConfig, + ) {} + + async init(): Promise { + const existing = await this.db.select({ id: agentProfiles.id }).from(agentProfiles).limit(1); + if (existing.length > 0) return; + + const builtins = buildBuiltinAgentProfiles(this.config).map((input, index) => { + const record = createAgentProfileRecord({ ...input, isActive: index === 0 }); + return { + id: record.id, + name: record.name, + description: record.description, + runtime: record.runtime, + provider: record.provider, + model: record.model, + tools: record.tools, + mode: record.mode, + extensions: record.extensions, + extraArgs: record.extraArgs, + isBuiltIn: record.isBuiltin, + isActive: record.isActive, + customCommandTemplate: record.customCommandTemplate, + }; + }); + + if (builtins.length > 0) { + await this.db.insert(agentProfiles).values(builtins); + } + } + + async list(): Promise { + const rows = await this.db.select().from(agentProfiles).orderBy(desc(agentProfiles.isActive), agentProfiles.name); + return rows.map(toRecord); + } + + async get(id: string): Promise { + const rows = await this.db.select().from(agentProfiles).where(eq(agentProfiles.id, id)).limit(1); + return rows[0] ? toRecord(rows[0]) : undefined; + } + + async getActive(): Promise { + const rows = await this.db.select().from(agentProfiles).where(eq(agentProfiles.isActive, true)).limit(1); + return rows[0] ? toRecord(rows[0]) : undefined; + } + + async save(input: AgentProfileInput, id?: string): Promise { + const clean = sanitizeAgentProfileInput(input); + const validation = validateAgentProfile(clean, this.config); + if (!validation.ok) { + throw new Error(validation.errors.join("; ")); + } + + const record = createAgentProfileRecord(clean); + const profileId = id ?? record.id; + const now = new Date(); + if (clean.isActive) { + await this.db.update(agentProfiles).set({ isActive: false, updatedAt: now }); + } + + await this.db.insert(agentProfiles).values({ + id: profileId, + name: clean.name, + description: clean.description, + runtime: clean.runtime, + provider: clean.provider, + model: clean.model, + tools: clean.tools ?? [], + mode: clean.mode, + extensions: clean.extensions ?? [], + extraArgs: clean.extraArgs, + isBuiltIn: Boolean(clean.isBuiltin), + isActive: Boolean(clean.isActive), + customCommandTemplate: clean.customCommandTemplate, + updatedAt: now, + }).onConflictDoUpdate({ + target: agentProfiles.id, + set: { + name: clean.name, + description: clean.description, + runtime: clean.runtime, + provider: clean.provider, + model: clean.model, + tools: clean.tools ?? [], + mode: clean.mode, + extensions: clean.extensions ?? [], + extraArgs: clean.extraArgs, + isActive: Boolean(clean.isActive), + customCommandTemplate: clean.customCommandTemplate, + updatedAt: now, + }, + }); + + if (!clean.isActive) { + const active = await this.getActive(); + if (!active) { + await this.setActive(profileId); + } + } + + const saved = await this.get(profileId); + if (!saved) { + throw new Error("Failed to load saved agent profile"); + } + return saved; + } + + async delete(id: string): Promise { + const existing = await this.get(id); + if (!existing || existing.isBuiltin) return false; + await this.db.delete(agentProfiles).where(eq(agentProfiles.id, id)); + if (existing.isActive) { + const replacement = await this.db.select().from(agentProfiles).orderBy(agentProfiles.name).limit(1); + if (replacement[0]) { + await this.setActive(replacement[0].id); + } + } + return true; + } + + async setActive(id: string): Promise { + await this.db.transaction(async (tx) => { + await tx.update(agentProfiles).set({ isActive: false, updatedAt: new Date() }); + await tx.update(agentProfiles).set({ isActive: true, updatedAt: new Date() }).where(eq(agentProfiles.id, id)); + }); + return this.get(id); + } + + async getEffectiveCommandTemplate(fallbackTemplate: string): Promise { + const active = await this.getActive(); + return resolveProfileCommandTemplate(active, fallbackTemplate); + } + + async count(): Promise { + const rows = await this.db.select({ count: sql`count(*)::int` }).from(agentProfiles); + return rows[0]?.count ?? 0; + } +} diff --git a/src/db/schema.ts b/src/db/schema.ts index 96bab25..edcf54d 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -273,3 +273,28 @@ export const configSections = pgTable("config_sections", { createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), }); + +export const agentProfiles = pgTable( + "agent_profiles", + { + id: uuid("id").primaryKey(), + name: text("name").notNull(), + description: text("description"), + runtime: text("runtime").notNull(), + provider: text("provider"), + model: text("model"), + tools: text("tools").array().notNull().default([]), + mode: text("mode"), + extensions: text("extensions").array().notNull().default([]), + extraArgs: text("extra_args"), + isBuiltIn: boolean("is_built_in").notNull().default(false), + isActive: boolean("is_active").notNull().default(false), + customCommandTemplate: text("custom_command_template"), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (t) => [ + index("agent_profiles_active_idx").on(t.isActive), + index("agent_profiles_runtime_idx").on(t.runtime), + ] +); diff --git a/src/index.ts b/src/index.ts index 3300265..cc483c7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,12 +28,14 @@ import { callLLMForJSON, type LLMCallerConfig } from "./llm/caller.js"; import { LearningStore } from "./observer/learning-store.js"; import { EvalStore } from "./eval/eval-store.js"; import { SetupStore } from "./db/setup-store.js"; +import { AgentProfileStore } from "./db/agent-profile-store.js"; // ── Service container ── interface Services { config: AppConfig; store: RunStore; + agentProfileStore: AgentProfileStore; githubService: GitHubService | undefined; memoryProvider: CemsProvider | undefined; hooks: RunLifecycleHooks; @@ -50,6 +52,8 @@ interface Services { async function createServices(config: AppConfig, db: Database): Promise { const store = new RunStore(db); await store.init(); + const agentProfileStore = new AgentProfileStore(db, config); + await agentProfileStore.init(); const githubService = GitHubService.create(config); const memoryProvider = config.cemsEnabled && config.cemsApiUrl && config.cemsApiKey @@ -106,7 +110,7 @@ async function createServices(config: AppConfig, db: Database): Promise { // 2. Load config (reads env vars, including any injected by wizard) const config = loadConfig(); - checkAgentDefault(config); // 3. One-time registrations (plugins) const pluginResult = await loadPlugins(getPluginDir()); @@ -150,6 +153,28 @@ async function main(): Promise { // 4. Create core services const svc = await createServices(config, db); + const activeAgentProfile = await svc.agentProfileStore.getActive(); + if (activeAgentProfile) { + config.agentCommandTemplate = await svc.agentProfileStore.getEffectiveCommandTemplate(config.baseAgentCommandTemplate ?? config.agentCommandTemplate); + config.activeAgentProfile = { + id: activeAgentProfile.id, + name: activeAgentProfile.name, + runtime: activeAgentProfile.runtime, + provider: activeAgentProfile.provider, + model: activeAgentProfile.model, + commandTemplate: config.agentCommandTemplate, + source: "profile", + }; + } else { + config.activeAgentProfile = { + id: "env-template", + name: "Raw AGENT_COMMAND_TEMPLATE", + runtime: "custom", + commandTemplate: config.agentCommandTemplate, + source: "env", + }; + } + checkAgentDefault(config); // 5. Recover stale in-progress runs from before restart const recoveredRuns = await svc.store.recoverInProgressRuns( @@ -245,6 +270,7 @@ async function main(): Promise { setTimeout(() => shutdown("WIZARD_COMPLETE"), 1000); }, svc.evalStore, + svc.agentProfileStore, ); } diff --git a/tests/agent-profile.test.ts b/tests/agent-profile.test.ts new file mode 100644 index 0000000..526e9eb --- /dev/null +++ b/tests/agent-profile.test.ts @@ -0,0 +1,169 @@ +import assert from "node:assert/strict"; +import { describe, test } from "node:test"; +import { buildBuiltinAgentProfiles, getAvailableProviders, renderAgentProfileTemplate, validateAgentProfile } from "../src/agent-profile.js"; +import type { AppConfig } from "../src/config.js"; + +function makeConfig(): AppConfig { + return { + appName: "test", + appSlug: "test", + slackCommandName: "/goose", + slackAllowedChannels: [], + repoAllowlist: [], + runnerConcurrency: 1, + workRoot: ".work", + dataDir: "data", + dryRun: false, + branchPrefix: "goose/", + defaultBaseBranch: "main", + gitAuthorName: "Test", + gitAuthorEmail: "test@example.com", + agentCommandTemplate: "echo test", + validationCommand: "", + lintFixCommand: "", + localTestCommand: "", + maxValidationRounds: 2, + agentTimeoutSeconds: 600, + slackProgressHeartbeatSeconds: 20, + dashboardEnabled: true, + dashboardHost: "127.0.0.1", + dashboardPort: 8787, + maxTaskChars: 4000, + workspaceCleanupEnabled: false, + workspaceMaxAgeHours: 24, + workspaceCleanupIntervalMinutes: 30, + cemsEnabled: false, + mcpExtensions: [], + piAgentExtensions: [], + pipelineFile: "pipelines/pipeline.yml", + observerEnabled: false, + observerAlertChannelId: "", + observerMaxRunsPerDay: 10, + observerMaxRunsPerRepoPerDay: 5, + observerCooldownMinutes: 10, + observerRulesFile: "observer-rules/default.yml", + observerRepoMap: new Map(), + observerSlackWatchedChannels: [], + observerSlackBotAllowlist: [], + observerSentryPollIntervalSeconds: 300, + observerWebhookPort: 9090, + observerWebhookSecrets: {}, + observerGithubPollIntervalSeconds: 300, + observerGithubWatchedRepos: [], + openrouterApiKey: "or-key", + anthropicApiKey: "anth-key", + openaiApiKey: "oa-key", + defaultLlmModel: "anthropic/claude-sonnet-4-6", + planTaskModel: "anthropic/claude-sonnet-4-6", + scopeJudgeEnabled: false, + scopeJudgeModel: "anthropic/claude-sonnet-4-6", + scopeJudgeMinPassScore: 60, + orchestratorModel: "openai/gpt-4.1-mini", + orchestratorTimeoutMs: 180000, + orchestratorWallClockTimeoutMs: 480000, + autonomousSchedulerEnabled: false, + autonomousSchedulerMaxDeferred: 100, + autonomousSchedulerIntervalMs: 300000, + observerSmartTriageEnabled: false, + observerSmartTriageModel: "anthropic/claude-sonnet-4-6", + observerSmartTriageTimeoutMs: 10000, + browserVerifyEnabled: false, + screenshotEnabled: false, + browserVerifyModel: "anthropic/claude-sonnet-4-6", + browserVerifyMaxSteps: 15, + browserVerifyExecTimeoutMs: 300000, + browserVerifyEmailDomains: [], + ciWaitEnabled: false, + ciPollIntervalSeconds: 30, + ciPatienceTimeoutSeconds: 120, + ciMaxWaitSeconds: 600, + ciCheckFilter: [], + ciMaxFixRounds: 3, + teamChannelMap: new Map(), + sandboxEnabled: false, + sandboxImage: "node:20-slim", + sandboxHostWorkPath: "", + sandboxCpus: 1, + sandboxMemoryMb: 512, + supervisorEnabled: false, + supervisorRunTimeoutSeconds: 3600, + supervisorNodeStaleSeconds: 600, + supervisorWatchdogIntervalSeconds: 30, + supervisorMaxAutoRetries: 2, + supervisorRetryCooldownSeconds: 60, + supervisorMaxRetriesPerDay: 5, + databaseUrl: "postgres://example/test", + }; +} + +describe("agent profile helpers", () => { + test("returns configured providers", () => { + const providers = getAvailableProviders(makeConfig()); + assert.equal(providers.filter((provider) => provider.configured).length, 3); + }); + + test("validates structured profiles against configured providers", () => { + const result = validateAgentProfile({ + name: "Pi OpenAI", + runtime: "pi", + provider: "openai", + model: "openai/gpt-4.1-mini", + tools: ["read", "write"], + }, makeConfig()); + assert.equal(result.ok, true); + assert.deepEqual(result.errors, []); + }); + + test("renders pi command template with placeholders", () => { + const command = renderAgentProfileTemplate({ + name: "Pi OpenAI", + runtime: "pi", + provider: "openai", + model: "openai/gpt-4.1-mini", + tools: ["read", "write"], + }); + assert.ok(command.includes("cd {{repo_dir}} && pi -p @{{prompt_file}}")); + assert.ok(command.includes("--model 'openai/gpt-4.1-mini'")); + assert.ok(command.includes("--tools 'read,write'")); + }); + + test("renders custom command template unchanged", () => { + const command = renderAgentProfileTemplate({ + name: "Custom", + runtime: "custom", + customCommandTemplate: "cd {{repo_dir}} && custom-agent @{{prompt_file}}", + }); + assert.equal(command, "cd {{repo_dir}} && custom-agent @{{prompt_file}}"); + }); + + test("does not render unsupported --tools flag for codex", () => { + const command = renderAgentProfileTemplate({ + name: "Codex", + runtime: "codex", + provider: "openai", + model: "gpt-5.4", + tools: ["read", "write", "edit", "bash"], + }); + assert.ok(command.includes("codex exec --full-auto --model 'gpt-5.4'")); + assert.ok(!command.includes("--tools")); + }); + + test("rejects codex profiles with openrouter provider", () => { + const result = validateAgentProfile({ + name: "Codex OpenRouter", + runtime: "codex", + provider: "openrouter", + model: "openai/gpt-5.4", + }, makeConfig()); + assert.equal(result.ok, false); + assert.ok(result.errors.includes("Runtime codex does not support provider openrouter")); + }); + + test("builds builtin profiles from configured api keys", () => { + const builtins = buildBuiltinAgentProfiles(makeConfig()); + assert.ok(builtins.some((profile) => profile.name === "Pi + OpenAI")); + assert.ok(builtins.some((profile) => profile.name === "Codex + OpenAI")); + assert.ok(builtins.some((profile) => profile.name === "Claude + Anthropic")); + assert.ok(!builtins.some((profile) => profile.name === "Codex + OpenRouter")); + }); +}); From 201c86895b1029b29b4f795dec810241725c9aa5 Mon Sep 17 00:00:00 2001 From: Vsevolod Avramov Date: Thu, 9 Apr 2026 14:41:09 +0300 Subject: [PATCH 2/3] Add agent profile dashboard wizard --- docs/agent-profile-wizard.md | 233 +++++++++ src/dashboard-server.ts | 225 +++++++++ src/dashboard/agent-profile-list-html.ts | 192 ++++++++ src/dashboard/agent-profile-wizard-html.ts | 527 +++++++++++++++++++++ src/dashboard/html.ts | 197 +++++++- 5 files changed, 1363 insertions(+), 11 deletions(-) create mode 100644 docs/agent-profile-wizard.md create mode 100644 src/dashboard/agent-profile-list-html.ts create mode 100644 src/dashboard/agent-profile-wizard-html.ts diff --git a/docs/agent-profile-wizard.md b/docs/agent-profile-wizard.md new file mode 100644 index 0000000..e9166b0 --- /dev/null +++ b/docs/agent-profile-wizard.md @@ -0,0 +1,233 @@ +# Agent Profile Wizard + +## Goal + +Replace the single raw `AGENT_COMMAND_TEMPLATE` env var as the primary configuration +mechanism with a structured "Agent Profile" system stored in the database and +editable from the web UI. + +The intent is to make agent configuration: + +- easier to understand +- easier to validate +- safer to edit +- less dependent on hand-written shell strings + +This should be conceptually similar to the existing `/setup` wizard: a guided UI +flow that helps the operator choose a valid configuration. + +## Why + +Today, the agent runtime is configured by a raw shell template, for example: + +```bash +cd {{repo_dir}} && pi -p @{{prompt_file}} --model openrouter/openai/gpt-4.1-mini --no-session --mode json --tools read,write,edit,bash,grep,find,ls +``` + +This has several problems: + +- easy to break quoting or argument structure +- hard to validate before saving +- hard to explain in the UI +- easy to create invalid combinations of runtime, provider, and model +- impossible to guide the user based on configured API keys + +## Core Idea + +Instead of storing only a raw template string, Gooseherd should store a structured +Agent Profile and derive the final command from it. + +An Agent Profile represents: + +- which CLI/runtime to use +- which provider to use +- which model to use +- which tools to enable +- optional extra runtime-specific settings + +The rendered command becomes a derived artifact, not the main source of truth. + +## First-Version Scope + +The first version should focus on a guided wizard and a minimal structured model. + +Suggested supported runtimes: + +- `pi` +- `codex` +- `claude` + +The first version should also keep an escape hatch: + +- `custom` profile type with a raw command template + +This avoids blocking uncommon or experimental setups while still moving the main +path toward structured configuration. + +## Agent Profile Data Model + +Suggested fields: + +- `id` +- `name` +- `description` +- `runtime` +- `provider` +- `model` +- `tools` +- `mode` +- `extensions` +- `extra_args` +- `is_builtin` +- `is_active` +- `custom_command_template` + +Notes: + +- `runtime` is the CLI, such as `pi` or `codex` +- `provider` is the API/backend provider, such as `openai` or `openrouter` +- `model` is the selected model id +- `tools` is the selected tool set for runtimes that support tool selection +- `custom_command_template` is only for the `custom` escape hatch + +## UI Direction + +Add a dedicated Agent Profile wizard in the web UI, similar in spirit to `/setup`. + +Suggested flow: + +1. Choose runtime +2. Choose provider +3. Choose model +4. Choose tools +5. Review generated command preview +6. Save profile and optionally mark it active/default + +The UI should explain what the profile does in plain language, not only show the + generated shell command. + +## Provider Availability + +The wizard should only offer providers that are actually configured in the current +environment. + +Examples: + +- If `OPENAI_API_KEY` is configured, show `openai` +- If `OPENROUTER_API_KEY` is not configured, do not offer `openrouter` +- If `ANTHROPIC_API_KEY` is configured, show `anthropic` + +This prevents users from building profiles that are guaranteed to fail at runtime. + +It is useful to distinguish: + +- provider configured or not configured +- model list available or not available +- runtime/provider combination supported or not supported + +## Online Model Loading + +For each selected provider, the wizard should load the available models online. + +This should be done live from the backend when the provider is selected. + +Proposed behavior: + +- frontend requests models for the selected provider +- backend loads the provider's current model catalog +- frontend shows the available models for selection +- user may still manually enter a model id if needed + +Important decision: + +- do not cache model-catalog results + +Rationale: + +- model availability changes often enough that stale data is misleading +- this is an operator-facing configuration flow, not a hot path +- correctness is more important than hiding an extra network call +- fallback to manual model entry is sufficient if the provider catalog request fails + +## Validation Rules + +The wizard should validate configuration before saving. + +Examples: + +- `pi + openai` is valid when `OPENAI_API_KEY` exists +- `pi + openrouter` is valid when `OPENROUTER_API_KEY` exists +- `pi + openrouter` should not even be selectable without that key +- unsupported runtime/provider pairs should be rejected by the backend even if the + UI somehow submits them + +Validation should happen in two places: + +- frontend for good UX +- backend for correctness + +## Command Rendering + +Each runtime should have its own renderer that converts an Agent Profile into the +final command string. + +Examples: + +- `pi` renderer +- `codex` renderer +- `claude` renderer + +This is preferable to one giant generic template builder because each CLI has its +own conventions and arguments. + +The generated command should be shown in the UI as a preview, but the profile data +should remain the canonical representation. + +## Built-In Profiles + +Gooseherd should likely ship with a small set of built-in profiles such as: + +- `pi + OpenAI` +- `pi + OpenRouter` +- `claude + Anthropic` + +Built-in profiles should include: + +- human-readable name +- short description +- reasonable default tools +- recommended default model where possible + +These profiles should help users get started quickly without writing commands by +hand. + +## API / Backend Shape + +Likely backend needs: + +- endpoint to list available providers based on current env +- endpoint to load models for a provider +- CRUD for agent profiles +- endpoint or service to render a profile into a command preview +- validation layer for runtime/provider/model/tool compatibility + +## Non-Goals For First Version + +- caching provider model catalogs +- abstracting every possible CLI at once +- removing the raw-template escape hatch +- making the UI depend entirely on provider model APIs + +## Summary + +The recommended direction is: + +- move from raw shell template editing to structured Agent Profiles +- add an Agent Profile wizard in the web UI +- only show configured providers +- load model lists online per provider without caching +- keep a `custom` fallback for advanced cases +- render final commands from structured profile data via runtime-specific renderers + +This should make agent configuration much more understandable and much harder to +misconfigure. diff --git a/src/dashboard-server.ts b/src/dashboard-server.ts index 1a8f3f9..d60a0c1 100644 --- a/src/dashboard-server.ts +++ b/src/dashboard-server.ts @@ -18,9 +18,20 @@ import type { PipelineStore } from "./pipeline/pipeline-store.js"; import type { LearningStore } from "./observer/learning-store.js"; import { SetupStore } from "./db/setup-store.js"; import { wizardHtml } from "./dashboard/wizard-html.js"; +import { agentProfileWizardHtml } from "./dashboard/agent-profile-wizard-html.js"; +import { agentProfileListHtml } from "./dashboard/agent-profile-list-html.js"; import { GitHubService } from "./github.js"; import type { EvalStore } from "./eval/eval-store.js"; import { loadScenariosFromDir } from "./eval/scenario-loader.js"; +import { AgentProfileStore } from "./db/agent-profile-store.js"; +import { + getAvailableProviders, + renderAgentProfileTemplate, + sanitizeAgentProfileInput, + validateAgentProfile, + type AgentProvider, + type AgentProfileInput, +} from "./agent-profile.js"; /** Lean interface — dashboard only reads observer state, never mutates it. */ export interface DashboardObserver { @@ -298,6 +309,85 @@ async function computeRunStats(store: RunStore) { return { totalRuns, completedRuns, failedRuns, successRate, totalCostUsd, avgCostUsd, runsLast24h }; } +async function syncActiveAgentProfileConfig(config: AppConfig, agentProfileStore: AgentProfileStore): Promise { + const fallbackTemplate = config.baseAgentCommandTemplate ?? config.agentCommandTemplate; + const active = await agentProfileStore.getActive(); + if (!active) { + config.agentCommandTemplate = fallbackTemplate; + config.activeAgentProfile = { + id: "env-template", + name: "Raw AGENT_COMMAND_TEMPLATE", + runtime: "custom", + commandTemplate: fallbackTemplate, + source: "env", + }; + return; + } + + const commandTemplate = await agentProfileStore.getEffectiveCommandTemplate(fallbackTemplate); + config.agentCommandTemplate = commandTemplate; + config.activeAgentProfile = { + id: active.id, + name: active.name, + runtime: active.runtime, + provider: active.provider, + model: active.model, + commandTemplate, + source: "profile", + }; +} + +async function loadProviderModels(config: AppConfig, provider: AgentProvider): Promise { + const unique = (values: string[]) => [...new Set(values.filter(Boolean))].sort((a, b) => a.localeCompare(b)); + + if (provider === "openai") { + if (!config.openaiApiKey) throw new Error("OPENAI_API_KEY is not configured"); + const response = await fetch("https://api.openai.com/v1/models", { + headers: { Authorization: `Bearer ${config.openaiApiKey}` }, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error(`OpenAI returned ${String(response.status)}`); + } + const data = await response.json() as { data?: Array<{ id?: string }> }; + return unique((data.data ?? []).map((entry) => entry.id?.trim()).filter((entry): entry is string => Boolean(entry))); + } + + if (provider === "anthropic") { + if (!config.anthropicApiKey) throw new Error("ANTHROPIC_API_KEY is not configured"); + const response = await fetch("https://api.anthropic.com/v1/models", { + headers: { + "x-api-key": config.anthropicApiKey, + "anthropic-version": "2023-06-01", + }, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error(`Anthropic returned ${String(response.status)}`); + } + const data = await response.json() as { data?: Array<{ id?: string }> }; + return unique((data.data ?? []).map((entry) => entry.id?.trim()).filter((entry): entry is string => Boolean(entry))); + } + + if (!config.openrouterApiKey) throw new Error("OPENROUTER_API_KEY is not configured"); + const response = await fetch("https://openrouter.ai/api/v1/models", { + headers: { Authorization: `Bearer ${config.openrouterApiKey}` }, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error(`OpenRouter returned ${String(response.status)}`); + } + const data = await response.json() as { data?: Array<{ id?: string }> }; + return unique((data.data ?? []).map((entry) => entry.id?.trim()).filter((entry): entry is string => Boolean(entry))); +} + +function decorateAgentProfile(profile: AgentProfileInput & { id?: string; isBuiltin?: boolean; isActive?: boolean; createdAt?: string; updatedAt?: string }): Record { + return { + ...profile, + commandTemplate: renderAgentProfileTemplate(profile), + }; +} + export function startDashboardServer( config: AppConfig, store: RunStore, @@ -309,6 +399,7 @@ export function startDashboardServer( setupStore?: SetupStore, onSetupComplete?: () => Promise, evalStore?: EvalStore, + agentProfileStore?: AgentProfileStore, ): void { const githubService = GitHubService.create(config); let githubRepositoriesCache: CachedGitHubRepositories | undefined; @@ -349,6 +440,16 @@ export function startDashboardServer( return; } + if (req.method === "GET" && pathname === "/agent-profiles") { + sendText(res, 200, agentProfileListHtml(config.appName), "text/html"); + return; + } + + if (req.method === "GET" && pathname === "/agent-profiles/new") { + sendText(res, 200, agentProfileWizardHtml(config.appName), "text/html"); + return; + } + if (req.method === "GET" && pathname === "/api/setup/status") { if (!setupStore) { sendJson(res, 501, { error: "Setup not available" }); return; } const status = await setupStore.getStatus(); @@ -557,6 +658,7 @@ export function startDashboardServer( if (req.method === "GET" && pathname === "/api/settings") { const githubAuthMode = resolveGitHubAuthMode(config); const stats = await computeRunStats(store); + const profiles = agentProfileStore ? (await agentProfileStore.list()).map((profile) => decorateAgentProfile(profile)) : []; sendJson(res, 200, { config: { @@ -584,12 +686,89 @@ export function startDashboardServer( browserVerify: config.browserVerifyModel, }, agentCommandTemplate: config.agentCommandTemplate, + activeAgentProfile: config.activeAgentProfile, + agentProfiles: profiles, }, stats, }); return; } + if (req.method === "GET" && pathname === "/api/agent-providers") { + sendJson(res, 200, { + providers: getAvailableProviders(config), + }); + return; + } + + if (req.method === "GET" && pathname === "/api/agent-models") { + const provider = requestUrl.searchParams.get("provider") as AgentProvider | null; + if (!provider) { + sendJson(res, 400, { error: "provider is required" }); + return; + } + try { + const models = await loadProviderModels(config, provider); + sendJson(res, 200, { provider, models }); + } catch (error) { + sendJson(res, 502, { error: error instanceof Error ? error.message : "Failed to load models", models: [] }); + } + return; + } + + if (req.method === "POST" && pathname === "/api/agent-profiles/preview") { + const raw = await readBody(req); + if (raw === null) { sendJson(res, 413, { error: "Request body too large" }); return; } + let parsed: AgentProfileInput; + try { + parsed = JSON.parse(raw) as AgentProfileInput; + } catch { + sendJson(res, 400, { error: "Invalid JSON body" }); + return; + } + const profile = sanitizeAgentProfileInput(parsed); + const validation = validateAgentProfile(profile, config); + sendJson(res, 200, { + ok: validation.ok, + errors: validation.errors, + commandTemplate: renderAgentProfileTemplate(profile), + }); + return; + } + + if (req.method === "GET" && pathname === "/api/agent-profiles") { + if (!agentProfileStore) { + sendJson(res, 501, { error: "Agent profiles are unavailable" }); + return; + } + sendJson(res, 200, { profiles: (await agentProfileStore.list()).map((profile) => decorateAgentProfile(profile)) }); + return; + } + + if (req.method === "POST" && pathname === "/api/agent-profiles") { + if (!agentProfileStore) { + sendJson(res, 501, { error: "Agent profiles are unavailable" }); + return; + } + const raw = await readBody(req); + if (raw === null) { sendJson(res, 413, { error: "Request body too large" }); return; } + let parsed: AgentProfileInput; + try { + parsed = JSON.parse(raw) as AgentProfileInput; + } catch { + sendJson(res, 400, { error: "Invalid JSON body" }); + return; + } + try { + const profile = await agentProfileStore.save(parsed); + await syncActiveAgentProfileConfig(config, agentProfileStore); + sendJson(res, 201, { ok: true, profile: decorateAgentProfile(profile) }); + } catch (error) { + sendJson(res, 400, { error: error instanceof Error ? error.message : "Failed to save agent profile" }); + } + return; + } + if (req.method === "GET" && pathname === "/api/github/repositories") { if (!githubService) { sendJson(res, 501, { error: "GitHub integration is not configured" }); @@ -710,6 +889,52 @@ export function startDashboardServer( } const parts = pathname.split("/").filter(Boolean); + if (parts[0] === "api" && parts[1] === "agent-profiles" && parts[2]) { + if (!agentProfileStore) { + sendJson(res, 501, { error: "Agent profiles are unavailable" }); + return; + } + const profileId = decodeURIComponent(parts[2]); + + if (parts.length === 3 && req.method === "PUT") { + const raw = await readBody(req); + if (raw === null) { sendJson(res, 413, { error: "Request body too large" }); return; } + let parsed: AgentProfileInput; + try { + parsed = JSON.parse(raw) as AgentProfileInput; + } catch { + sendJson(res, 400, { error: "Invalid JSON body" }); + return; + } + try { + const profile = await agentProfileStore.save(parsed, profileId); + await syncActiveAgentProfileConfig(config, agentProfileStore); + sendJson(res, 200, { ok: true, profile: decorateAgentProfile(profile) }); + } catch (error) { + sendJson(res, 400, { error: error instanceof Error ? error.message : "Failed to update agent profile" }); + } + return; + } + + if (parts.length === 3 && req.method === "DELETE") { + const deleted = await agentProfileStore.delete(profileId); + await syncActiveAgentProfileConfig(config, agentProfileStore); + sendJson(res, deleted ? 200 : 404, deleted ? { ok: true } : { error: "Profile not found or cannot be deleted" }); + return; + } + + if (parts.length === 4 && parts[3] === "activate" && req.method === "POST") { + const profile = await agentProfileStore.setActive(profileId); + if (!profile) { + sendJson(res, 404, { error: "Profile not found" }); + return; + } + await syncActiveAgentProfileConfig(config, agentProfileStore); + sendJson(res, 200, { ok: true, profile: decorateAgentProfile(profile) }); + return; + } + } + if (parts[0] === "api" && parts[1] === "runs" && parts[2]) { const id = decodeURIComponent(parts[2]); const run = await store.findRunByIdentifier(id); diff --git a/src/dashboard/agent-profile-list-html.ts b/src/dashboard/agent-profile-list-html.ts new file mode 100644 index 0000000..58918da --- /dev/null +++ b/src/dashboard/agent-profile-list-html.ts @@ -0,0 +1,192 @@ +import { escapeHtml } from "./html.js"; + +export function agentProfileListHtml(appName: string): string { + return ` + + + + + ${escapeHtml(appName)} - Agent Profiles + + + + + + +
+
+
+

Agent Profiles

+
Choose which structured profile renders the effective agent command, or create a new one through the wizard.
+
+
+ Back + New Profile Wizard +
+
+
+
Effective Agent Command
+
Loading...
+ +
Profiles
+
Loading...
+
+
+ + + +`; +} diff --git a/src/dashboard/agent-profile-wizard-html.ts b/src/dashboard/agent-profile-wizard-html.ts new file mode 100644 index 0000000..34e8795 --- /dev/null +++ b/src/dashboard/agent-profile-wizard-html.ts @@ -0,0 +1,527 @@ +import { escapeHtml } from "./html.js"; + +export function agentProfileWizardHtml(appName: string): string { + return ` + + + + + ${escapeHtml(appName)} - Agent Profile Wizard + + + + + + +
+

Agent Profile Wizard

+
Create a structured agent profile. The shell command is derived from runtime, provider, model, and tool choices.
+ +
+
+
+
+
+
+
+ +
+

Choose Runtime

+

Select the CLI/runtime that will execute coding tasks.

+
+ + + + +
+
+ + +
+
+ +
+

Choose Provider

+

Only configured providers are selectable. Unsupported runtime/provider pairs are hidden.

+
+
+ +
+ + +
+
+ +
+

Choose Model

+

The wizard loads the live model catalog from the backend. Manual entry still works.

+
+
+ +
+ + + + +
+ +
+ + + +
+
+ +
+

Tools & Command Shape

+

Define tools and optional runtime-specific extras, or use a raw command for the custom escape hatch.

+
+
+
+ + +
Comma-separated. For Claude this maps to allowed tools. For Pi it maps to runtime tool flags.
+
+ + +
Mode is a runtime-specific extra. For example, Pi can use it to append an additional mode flag. Leave it empty unless your chosen CLI expects one.
+ + + + +
+ + +
+ + +
+
+ +
+

Review & Save

+

Review the structured profile and generated command preview before saving.

+
+
+
+
Loading preview...
+
+ + + +
+
+
+ + + +`; +} diff --git a/src/dashboard/html.ts b/src/dashboard/html.ts index cf7ba59..ae3e615 100644 --- a/src/dashboard/html.ts +++ b/src/dashboard/html.ts @@ -3064,29 +3064,198 @@ export function dashboardHtml(config: AppConfig): string { refreshSelected().catch(console.error); }; // ── Settings slide-over ── - el.settingsBtn.onclick = async () => { - el.settingsOverlay.classList.add('open'); + function settingsBadge(on) { + return '' + (on ? 'On' : 'Off') + ''; + } + + async function loadAgentModels(provider, modelSelect, statusEl, manualInput) { + if (!provider) { + modelSelect.innerHTML = ''; + return; + } + statusEl.textContent = 'Loading model catalog...'; + try { + var data = await fetchJson('/api/agent-models?provider=' + encodeURIComponent(provider)); + var models = Array.isArray(data.models) ? data.models : []; + modelSelect.innerHTML = ''; + for (var i = 0; i < models.length; i++) { + var opt = document.createElement('option'); + opt.value = models[i]; + opt.textContent = models[i]; + modelSelect.appendChild(opt); + } + statusEl.textContent = models.length ? 'Loaded ' + models.length + ' models.' : 'Catalog returned no models. Manual entry still works.'; + if (manualInput.value) { + modelSelect.value = manualInput.value; + } + } catch (err) { + modelSelect.innerHTML = ''; + statusEl.textContent = 'Could not load live catalog. Enter model id manually.'; + } + } + + function collectAgentProfileForm() { + var providerSelect = document.getElementById('ap-provider'); + var toolsInput = document.getElementById('ap-tools'); + var extensionsInput = document.getElementById('ap-extensions'); + return { + name: (document.getElementById('ap-name')?.value || '').trim(), + description: (document.getElementById('ap-description')?.value || '').trim(), + runtime: document.getElementById('ap-runtime')?.value || 'pi', + provider: providerSelect ? providerSelect.value || undefined : undefined, + model: (document.getElementById('ap-model-manual')?.value || document.getElementById('ap-model-select')?.value || '').trim(), + tools: (toolsInput?.value || '').split(',').map(function(v) { return v.trim(); }).filter(Boolean), + mode: (document.getElementById('ap-mode')?.value || '').trim() || undefined, + extensions: (extensionsInput?.value || '').split(',').map(function(v) { return v.trim(); }).filter(Boolean), + extraArgs: (document.getElementById('ap-extra-args')?.value || '').trim() || undefined, + customCommandTemplate: (document.getElementById('ap-custom-command')?.value || '').trim() || undefined, + isActive: !!document.getElementById('ap-is-active')?.checked + }; + } + + function buildProfileCard(profile, activeId) { + var meta = [profile.runtime, profile.provider, profile.model].filter(Boolean).join(' / '); + var actions = '
'; + if (profile.id !== activeId) { + actions += ''; + } else { + actions += 'Active'; + } + if (!profile.isBuiltin) { + actions += ''; + } + actions += '
'; + return '
' + + '
' + + '
' + esc(profile.name || '') + '
' + + '
' + esc(profile.description || '') + '
' + + '
' + esc(meta || 'custom') + '
' + + '
' + + '
Command preview
' + + '
' + esc(profile.customCommandTemplate || profile.commandTemplate || '') + '
' + + actions + + '
'; + } + + function bindSettingsAgentProfileActions() { + var providerSelect = document.getElementById('ap-provider'); + var runtimeSelect = document.getElementById('ap-runtime'); + var modelSelect = document.getElementById('ap-model-select'); + var modelManual = document.getElementById('ap-model-manual'); + var statusEl = document.getElementById('ap-model-status'); + var customWrap = document.getElementById('ap-custom-wrap'); + var structuredWrap = document.getElementById('ap-structured-wrap'); + var previewEl = document.getElementById('ap-preview'); + var submitBtn = document.getElementById('ap-save'); + var previewBtn = document.getElementById('ap-preview-btn'); + var refreshBtn = document.getElementById('ap-refresh-models'); + + async function refreshModels() { + if (providerSelect && modelSelect && statusEl && modelManual) { + await loadAgentModels(providerSelect.value, modelSelect, statusEl, modelManual); + } + } + + function syncRuntimeUi() { + var isCustom = runtimeSelect && runtimeSelect.value === 'custom'; + if (customWrap) customWrap.style.display = isCustom ? 'block' : 'none'; + if (structuredWrap) structuredWrap.style.display = isCustom ? 'none' : 'block'; + } + + if (runtimeSelect) { + runtimeSelect.onchange = function() { + syncRuntimeUi(); + }; + syncRuntimeUi(); + } + if (providerSelect) { + providerSelect.onchange = function() { + refreshModels().catch(console.error); + }; + } + if (modelSelect && modelManual) { + modelSelect.onchange = function() { + if (modelSelect.value) modelManual.value = modelSelect.value; + }; + } + if (refreshBtn) { + refreshBtn.onclick = function() { + refreshModels().catch(console.error); + }; + } + if (previewBtn) { + previewBtn.onclick = async function() { + previewEl.textContent = 'Generating preview...'; + var payload = collectAgentProfileForm(); + try { + var result = await fetchJson('/api/agent-profiles/preview', { method: 'POST', body: JSON.stringify(payload) }); + previewEl.textContent = (result.errors && result.errors.length ? 'Validation: ' + result.errors.join('; ') + '\\n\\n' : '') + (result.commandTemplate || ''); + } catch (err) { + previewEl.textContent = 'Failed to build preview.'; + } + }; + } + if (submitBtn) { + submitBtn.onclick = async function() { + var payload = collectAgentProfileForm(); + submitBtn.disabled = true; + try { + await fetchJson('/api/agent-profiles', { method: 'POST', body: JSON.stringify(payload) }); + await refreshSettingsPanel(); + } catch (err) { + previewEl.textContent = err && err.message ? err.message : 'Failed to save profile.'; + } finally { + submitBtn.disabled = false; + } + }; + } + + el.settingsBody.querySelectorAll('[data-ap-activate]').forEach(function(btn) { + btn.onclick = async function() { + var id = btn.getAttribute('data-ap-activate'); + if (!id) return; + await fetchJson('/api/agent-profiles/' + encodeURIComponent(id) + '/activate', { method: 'POST' }); + await refreshSettingsPanel(); + }; + }); + el.settingsBody.querySelectorAll('[data-ap-delete]').forEach(function(btn) { + btn.onclick = async function() { + var id = btn.getAttribute('data-ap-delete'); + if (!id || !confirm('Delete this profile?')) return; + await fetchJson('/api/agent-profiles/' + encodeURIComponent(id), { method: 'DELETE' }); + await refreshSettingsPanel(); + }; + }); + + refreshModels().catch(function() {}); + } + + async function refreshSettingsPanel() { el.settingsBody.textContent = 'Loading...'; try { var data = await fetchJson('/api/settings'); var c = data.config || {}; var s = data.stats || {}; - var badge = (on) => '' + (on ? 'On' : 'Off') + ''; el.settingsBody.innerHTML = '

Configuration

' + '
App Name' + esc(c.appName || '') + '
' + '
Pipeline' + esc(c.pipelineFile || '') + '
' + - '
Slack' + badge(c.slackConnected) + '
' + + '
Slack' + settingsBadge(c.slackConnected) + '
' + '
GitHub Auth' + esc(c.githubAuthMode || 'none') + '
' + - '
Agent Command' + esc(c.agentCommandTemplate || '') + '
' + + '
' + + '
' + + 'Agent Command' + + 'Edit' + + '
' + + '' + esc(c.agentCommandTemplate || '') + '
' + '
' + '

Features

' + - '
Observer' + badge(c.features?.observer) + '
' + - '
Sandbox' + badge(c.features?.sandbox) + '
' + - '
Browser Verify' + badge(c.features?.browserVerify) + '
' + - '
Scope Judge' + badge(c.features?.scopeJudge) + '
' + - '
CI Wait' + badge(c.features?.ciWait) + '
' + - '
Dry Run' + badge(c.features?.dryRun) + '
' + + '
Observer' + settingsBadge(c.features?.observer) + '
' + + '
Sandbox' + settingsBadge(c.features?.sandbox) + '
' + + '
Browser Verify' + settingsBadge(c.features?.browserVerify) + '
' + + '
Scope Judge' + settingsBadge(c.features?.scopeJudge) + '
' + + '
CI Wait' + settingsBadge(c.features?.ciWait) + '
' + + '
Dry Run' + settingsBadge(c.features?.dryRun) + '
' + '
' + '

Models

' + '
Default' + esc(c.models?.default || '') + '
' + @@ -3105,9 +3274,15 @@ export function dashboardHtml(config: AppConfig): string { '
' + 'Reconfigure' + '
'; + bindSettingsAgentProfileActions(); } catch (e) { el.settingsBody.textContent = 'Failed to load settings.'; } + } + + el.settingsBtn.onclick = async () => { + el.settingsOverlay.classList.add('open'); + await refreshSettingsPanel(); }; el.settingsClose.onclick = () => el.settingsOverlay.classList.remove('open'); el.settingsOverlay.onclick = (e) => { if (e.target === el.settingsOverlay) el.settingsOverlay.classList.remove('open'); }; From 143c0a9fb3694c094875832377c106caefb089a5 Mon Sep 17 00:00:00 2001 From: Vsevolod Avramov Date: Thu, 9 Apr 2026 14:41:18 +0300 Subject: [PATCH 3/3] Fix Codex agent runtime plumbing --- src/db/setup-store.ts | 1 + src/pipeline/nodes/implement.ts | 21 ++++++++++------ src/pipeline/pipeline-engine.ts | 2 +- tests/implement.test.ts | 12 +++++++++ tests/pipeline-engine.test.ts | 44 +++++++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/src/db/setup-store.ts b/src/db/setup-store.ts index 7742773..66f1363 100644 --- a/src/db/setup-store.ts +++ b/src/db/setup-store.ts @@ -245,6 +245,7 @@ export class SetupStore { const apiKey = stringOrUndefined(llm.secrets.apiKey); setEnvValue("ANTHROPIC_API_KEY", provider === "anthropic" ? apiKey : undefined); setEnvValue("OPENAI_API_KEY", provider === "openai" ? apiKey : undefined); + setEnvValue("CODEX_API_KEY", provider === "openai" ? apiKey : undefined); setEnvValue("OPENROUTER_API_KEY", provider === "openrouter" ? apiKey : undefined); setEnvValue("DEFAULT_LLM_MODEL", stringOrUndefined(llm.config.defaultModel)); } diff --git a/src/pipeline/nodes/implement.ts b/src/pipeline/nodes/implement.ts index 41074d9..e384edc 100644 --- a/src/pipeline/nodes/implement.ts +++ b/src/pipeline/nodes/implement.ts @@ -12,6 +12,8 @@ export interface AgentAnalysis { signals: string[]; } +const INTERNAL_GENERATED_FILES = new Set(["AGENTS.md"]); + /** * Implement node: run the coding agent. */ @@ -117,26 +119,31 @@ export async function analyzeAgentOutput( const filesChanged = namesResult.stdout.trim() ? namesResult.stdout.trim().split("\n") : []; + const meaningfulFilesChanged = filesChanged.filter((file) => !INTERNAL_GENERATED_FILES.has(file)); // Parse numstat for added/removed lines let totalAdded = 0; let totalRemoved = 0; for (const line of numstatResult.stdout.trim().split("\n")) { const match = line.match(/^(\d+)\s+(\d+)\s+/); + const file = line.split("\t")[2]; + if (file && INTERNAL_GENERATED_FILES.has(file)) { + continue; + } if (match) { totalAdded += parseInt(match[1]!, 10); totalRemoved += parseInt(match[2]!, 10); } } - const diffStats = { added: totalAdded, removed: totalRemoved, filesCount: filesChanged.length }; + const diffStats = { added: totalAdded, removed: totalRemoved, filesCount: meaningfulFilesChanged.length }; // 2. Garbage detection - if (filesChanged.length === 0) { + if (meaningfulFilesChanged.length === 0) { signals.push("no file changes detected"); return { verdict: "empty", - filesChanged, + filesChanged: meaningfulFilesChanged, diffSummary: statResult.stdout.trim(), diffStats, signals @@ -144,11 +151,11 @@ export async function analyzeAgentOutput( } // Mass deletion check: removed > 100 lines AND removed > 5x added AND > 5 files - if (totalRemoved > 100 && totalRemoved > totalAdded * 5 && filesChanged.length > 5) { - signals.push(`mass deletion detected: +${String(totalAdded)} -${String(totalRemoved)} across ${String(filesChanged.length)} files`); + if (totalRemoved > 100 && totalRemoved > totalAdded * 5 && meaningfulFilesChanged.length > 5) { + signals.push(`mass deletion detected: +${String(totalAdded)} -${String(totalRemoved)} across ${String(meaningfulFilesChanged.length)} files`); return { verdict: "suspect", - filesChanged, + filesChanged: meaningfulFilesChanged, diffSummary: statResult.stdout.trim(), diffStats, signals @@ -172,7 +179,7 @@ export async function analyzeAgentOutput( return { verdict: "clean", - filesChanged, + filesChanged: meaningfulFilesChanged, diffSummary: statResult.stdout.trim(), diffStats, signals diff --git a/src/pipeline/pipeline-engine.ts b/src/pipeline/pipeline-engine.ts index dee9b7e..b9c8edb 100644 --- a/src/pipeline/pipeline-engine.ts +++ b/src/pipeline/pipeline-engine.ts @@ -628,7 +628,7 @@ export class PipelineEngine { // Pass through agent-relevant env vars for (const key of [ - "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", + "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "CODEX_API_KEY", "XAI_API_KEY", "GEMINI_API_KEY", "CEMS_API_URL", "CEMS_API_KEY", "OPENROUTER_PROVIDER_PREFERENCES" diff --git a/tests/implement.test.ts b/tests/implement.test.ts index cd23b97..c60e671 100644 --- a/tests/implement.test.ts +++ b/tests/implement.test.ts @@ -38,6 +38,18 @@ test("analyzeAgentOutput: no changes → verdict empty", async (t) => { assert.ok(result.signals.some(s => s.includes("no file changes"))); }); +test("analyzeAgentOutput: AGENTS.md only → verdict empty", async (t) => { + const { dir, logFile, cleanup } = await makeGitRepo(); + t.after(cleanup); + + await writeFile(path.join(dir, "AGENTS.md"), "# AGENTS.md\n", "utf8"); + + const result = await analyzeAgentOutput(dir, "", "", logFile); + assert.equal(result.verdict, "empty"); + assert.deepEqual(result.filesChanged, []); + assert.equal(result.diffStats.filesCount, 0); +}); + test("analyzeAgentOutput: normal changes → verdict clean", async (t) => { const { dir, logFile, cleanup } = await makeGitRepo(); t.after(cleanup); diff --git a/tests/pipeline-engine.test.ts b/tests/pipeline-engine.test.ts index eb565d0..317b3c8 100644 --- a/tests/pipeline-engine.test.ts +++ b/tests/pipeline-engine.test.ts @@ -150,6 +150,50 @@ test("PipelineEngine: unified pipeline is the single pipeline file", () => { assert.equal(resolved, "pipelines/pipeline.yml"); }); +test("PipelineEngine: passes CODEX_API_KEY into sandbox env", async (t) => { + const tmpDir = await mkdtemp(path.join(os.tmpdir(), "pe-sandbox-env-")); + const workDir = path.join(tmpDir, "work"); + const runId = "test-run-sandbox-env"; + const runDir = path.join(workDir, runId); + const logFile = path.join(runDir, "run.log"); + await mkdir(runDir, { recursive: true }); + await writeFile(logFile, "", "utf8"); + + const savedOpenAi = process.env.OPENAI_API_KEY; + const savedCodex = process.env.CODEX_API_KEY; + process.env.OPENAI_API_KEY = "sk-openai-test"; + process.env.CODEX_API_KEY = "sk-codex-test"; + + t.after(async () => { + if (savedOpenAi === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = savedOpenAi; + if (savedCodex === undefined) delete process.env.CODEX_API_KEY; + else process.env.CODEX_API_KEY = savedCodex; + await rm(tmpDir, { recursive: true, force: true }); + }); + + let capturedEnv: Record | undefined; + const fakeContainerManager = { + createSandbox: async (_runId: string, sandboxConfig: { env: Record }) => { + capturedEnv = sandboxConfig.env; + return { containerId: "sandbox-1", containerName: "sandbox-1" }; + } + }; + + const config = makeConfig({ + workRoot: workDir, + sandboxEnabled: true, + sandboxHostWorkPath: workDir, + }); + const engine = new PipelineEngine(config, undefined, undefined, fakeContainerManager as never); + + await (engine as unknown as { buildAndCreateSandbox: (runId: string, image: string, logFile: string) => Promise }) + .buildAndCreateSandbox(runId, config.sandboxImage, logFile); + + assert.equal(capturedEnv?.OPENAI_API_KEY, "sk-openai-test"); + assert.equal(capturedEnv?.CODEX_API_KEY, "sk-codex-test"); +}); + test("PipelineEngine: auto-enables decide_recovery when browser_verify is enabled", async (t) => { const tmpDir = await mkdtemp(path.join(os.tmpdir(), "pe-auto-enable-")); const workDir = path.join(tmpDir, "work");