Update Agent Evals to run in a home directory (#9427)

Author: samedson

scripts/agent-evals/src/runner/gemini-cli-runner.ts

@@ -10,11 +10,12 @@ import {
 } from "./tool-matcher.js";
 import fs from "fs";
 import { throwFailure } from "./logging.js";
-import { getAgentEvalsRoot } from "./paths.js";
+import { getAgentEvalsRoot, RunDirectories } from "./paths.js";
 import { execSync } from "node:child_process";
 import { ToolMockName } from "../mock/tool-mocks.js";
 
 const READY_PROMPT = "Type your message";
+const INSTALL_ID = "238efa5b-efb2-44bd-9dce-9b081532681c";
 
 interface ParsedTelemetryLog {
   attributes?: {
@@ -44,15 +45,16 @@ export class GeminiCliRunner implements AgentTestRunner {
 
   constructor(
     private readonly testName: string,
-    testDir: string,
-    runDir: string,
+    dirs: RunDirectories,
     toolMocks: ToolMockName[],
   ) {
     // Create a settings file to point the CLI to a local telemetry log
-    this.telemetryPath = path.join(testDir, "telemetry.log");
+    this.telemetryPath = path.join(dirs.testDir, "telemetry.log");
     const mockPath = path.resolve(path.join(getAgentEvalsRoot(), "lib/mock/mock-tools-main.js"));
     const firebasePath = execSync("which firebase").toString().trim();
-    const settings = {
+
+    // Write workspace Gemini Settings
+    this.writeGeminiSettings(dirs.runDir, {
       general: {
         disableAutoUpdate: true,
       },
@@ -71,15 +73,29 @@ export class GeminiCliRunner implements AgentTestRunner {
           },
         },
       },
-    };
-    const geminiDir = path.join(runDir, ".gemini");
-    mkdirSync(geminiDir, { recursive: true });
-    writeFileSync(path.join(geminiDir, "settings.json"), JSON.stringify(settings, null, 2));
+    });
+
+    // Write user Gemini Settings
+    this.writeGeminiSettings(dirs.userDir, {
+      security: {
+        auth: {
+          selectedType: "gemini-api-key",
+        },
+      },
+      hasSeenIdeIntegrationNudge: true,
+    });
+
+    this.writeGeminiInstallId(dirs.userDir);
 
     this.cli = new InteractiveCLI("gemini", ["--yolo"], {
-      cwd: runDir,
+      cwd: dirs.runDir,
       readyPrompt: READY_PROMPT,
       showOutput: true,
+      env: {
+        // Overwrite $HOME so that we can support GCLI features that only apply
+        // on a per-user basis, like memories and extensions
+        HOME: dirs.userDir,
+      },
     });
   }
 
@@ -101,6 +117,21 @@ export class GeminiCliRunner implements AgentTestRunner {
     await this.cli.kill();
   }
 
+  writeGeminiSettings(dir: string, settings: any) {
+    const geminiDir = path.join(dir, ".gemini");
+    mkdirSync(geminiDir, { recursive: true });
+    writeFileSync(path.join(geminiDir, "settings.json"), JSON.stringify(settings, null, 2));
+  }
+
+  /**
+   * Writes a constant, real install ID so that we don't bump Gemini metrics
+   * with fake users
+   */
+  writeGeminiInstallId(userDir: string) {
+    const geminiDir = path.join(userDir, ".gemini");
+    writeFileSync(path.join(geminiDir, "installation_id"), INSTALL_ID);
+  }
+
   /**
    * Reads the agent's telemetry file and looks for the given event. Throws if
    * the event is not found

scripts/agent-evals/src/runner/index.ts

@@ -3,18 +3,18 @@ import { randomBytes } from "node:crypto";
 import { mkdirSync } from "node:fs";
 import { AgentTestRunner } from "./agent-test-runner.js";
 import { GeminiCliRunner } from "./gemini-cli-runner.js";
-import { buildFirebaseCli, clearUserMcpServers } from "./setup.js";
+import { buildFirebaseCli } from "./setup.js";
 import { addCleanup } from "../helpers/cleanup.js";
 import { TemplateName, copyTemplate, buildTemplates } from "../template/index.js";
 import { ToolMockName } from "../mock/tool-mocks.js";
+import { RunDirectories } from "./paths.js";
 
 export * from "./agent-test-runner.js";
 
 const dateName = new Date().toISOString().replace("T", "_").replace(/:/g, "-").replace(".", "-");
 
 export async function setupEnvironment(): Promise<void> {
   await buildFirebaseCli();
-  await clearUserMcpServers();
   await buildTemplates();
 }
 
@@ -35,13 +35,13 @@ export async function startAgentTest(
     throw new Error("startAgentTest must be called inside of an `it` block of a Mocha test.");
   }
   const testName = mocha.test.fullTitle();
-  const { testDir, runDir } = createRunDirectory(testName);
+  const dirs = createRunDirectory(testName);
 
   if (options?.templateName) {
-    copyTemplate(options.templateName, runDir);
+    copyTemplate(options.templateName, dirs.runDir);
   }
 
-  const run = new GeminiCliRunner(testName, testDir, runDir, options?.toolMocks || []);
+  const run = new GeminiCliRunner(testName, dirs, options?.toolMocks || []);
   await run.waitForReadyPrompt();
 
   addCleanup(async () => {
@@ -51,12 +51,17 @@ export async function startAgentTest(
   return run;
 }
 
-function createRunDirectory(testName: string): { testDir: string; runDir: string } {
+function createRunDirectory(testName: string): RunDirectories {
   const sanitizedName = testName.toLowerCase().replace(/[^a-z0-9]/g, "-");
   const testDir = path.resolve(
     path.join("output", dateName, `${sanitizedName}-${randomBytes(8).toString("hex")}`),
   );
+
   const runDir = path.join(testDir, "repo");
   mkdirSync(runDir, { recursive: true });
-  return { testDir, runDir };
+
+  const userDir = path.join(testDir, "user");
+  mkdirSync(userDir, { recursive: true });
+
+  return { testDir, runDir, userDir };
 }

scripts/agent-evals/src/runner/paths.ts

@@ -1,6 +1,8 @@
 import path from "path";
 import { fileURLToPath } from "url";
 
+export type RunDirectories = { testDir: string; runDir: string; userDir: string };
+
 export function getAgentEvalsRoot(): string {
   const thisFilePath = path.dirname(fileURLToPath(import.meta.url));
   return path.resolve(path.join(thisFilePath, "..", ".."));

scripts/agent-evals/src/runner/setup.ts

@@ -13,17 +13,3 @@ export async function buildFirebaseCli() {
   console.log(`Building Firebase CLI at ${firebaseCliRoot}`);
   await execPromise("./scripts/clean-install.sh", { cwd: firebaseCliRoot });
 }
-
-export async function clearUserMcpServers() {
-  console.log(`Clearing existing MCP servers...`);
-  try {
-    await execPromise("gemini extensions uninstall firebase");
-  } catch (_: any) {
-    /* This can fail if there's nothing installed, so ignore that */
-  }
-  try {
-    await execPromise("gemini mcp remove firebase");
-  } catch (_: any) {
-    /* This can fail if there's nothing installed, so ignore that */
-  }
-}