From 9eaae7a4278219190e46333ad19eef4c36e39a25 Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Thu, 23 Oct 2025 17:58:05 -0400 Subject: [PATCH 01/12] Add export/import of app distribution test cases as YAML. --- package.json | 1 + src/appdistribution/client.spec.ts | 89 ++++++++++++- src/appdistribution/client.ts | 58 +++++++++ src/appdistribution/types.ts | 35 ++++++ src/appdistribution/yaml_helper.spec.ts | 119 ++++++++++++++++++ src/appdistribution/yaml_helper.ts | 94 ++++++++++++++ .../appdistribution-testcases-export.ts | 30 +++++ .../appdistribution-testcases-import.ts | 30 +++++ src/commands/index.ts | 3 + 9 files changed, 458 insertions(+), 1 deletion(-) create mode 100644 src/appdistribution/yaml_helper.spec.ts create mode 100644 src/appdistribution/yaml_helper.ts create mode 100644 src/commands/appdistribution-testcases-export.ts create mode 100644 src/commands/appdistribution-testcases-import.ts diff --git a/package.json b/package.json index 2539ebfd5a6..2795e35480a 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "mocha": "nyc --reporter=html mocha 'src/**/*.spec.{ts,js}'", "prepare": "npm run clean && npm run build:publish", "test": "npm run lint:quiet && npm run test:compile && npm run mocha", + "test:appdistribution": "npm run lint:quiet && npm run test:compile && nyc mocha 'src/appdistribution/*.spec.{ts,js}'", "test:apptesting": "npm run lint:quiet && npm run test:compile && nyc mocha 'src/apptesting/*.spec.{ts,js}'", "test:client-integration": "bash ./scripts/client-integration-tests/run.sh", "test:compile": "tsc --project tsconfig.compile.json", diff --git a/src/appdistribution/client.spec.ts b/src/appdistribution/client.spec.ts index 8a24a84161f..c95f1cf91d0 100644 --- a/src/appdistribution/client.spec.ts +++ b/src/appdistribution/client.spec.ts @@ -7,7 +7,7 @@ import * as sinon from "sinon"; import * as tmp from "tmp"; import { AppDistributionClient } from "./client"; -import { BatchRemoveTestersResponse, Group, TestDevice } from "./types"; +import { BatchRemoveTestersResponse, Group, TestCase, TestDevice } from "./types"; import { appDistributionOrigin } from "../api"; import { Distribution } from "./distribution"; import { FirebaseError } from "../error"; @@ -501,4 +501,91 @@ describe("distribution", () => { expect(nock.isDone()).to.be.true; }); }); + + describe("listTestCases", () => { + it("should throw error if request fails", async () => { + nock(appDistributionOrigin()) + .get(`/v1alpha/${appName}/testCases`) + .reply(400, { error: { status: "FAILED_PRECONDITION" } }); + await expect(appDistributionClient.listTestCases(appName)).to.be.rejectedWith( + FirebaseError, + "Client failed to list test cases", + ); + expect(nock.isDone()).to.be.true; + }); + + it("should resolve with array of test cases when request succeeds", async () => { + const testCases: TestCase[] = [ + { + name: `$appName/testCases/tc_1`, + displayName: "Test Case 1", + aiInstructions: { + steps: [ + { + goal: "Win at all costs", + }, + ], + }, + }, + { + name: `$appName/testCases/tc_2`, + displayName: "Test Case 2", + aiInstructions: { steps: [] }, + }, + ]; + + nock(appDistributionOrigin()).get(`/v1alpha/${appName}/testCases`).reply(200, { + testCases: testCases, + }); + await expect(appDistributionClient.listTestCases(appName)).to.eventually.deep.eq(testCases); + expect(nock.isDone()).to.be.true; + }); + }); + + describe("createTestCase", () => { + const mockTestCase = { displayName: "Case", aiInstructions: { steps: [] } }; + + it("should throw error if request fails", async () => { + nock(appDistributionOrigin()) + .post(`/v1alpha/${appName}/testCases`) + .reply(400, { error: { status: "FAILED_PRECONDITION" } }); + await expect(appDistributionClient.createTestCase(appName, mockTestCase)).to.be.rejectedWith( + FirebaseError, + "Failed to create test case", + ); + expect(nock.isDone()).to.be.true; + }); + + it("should resolve with TestCase when request succeeds", async () => { + nock(appDistributionOrigin()).post(`/v1alpha/${appName}/testCases`).reply(200, mockTestCase); + await expect( + appDistributionClient.createTestCase(appName, mockTestCase), + ).to.be.eventually.deep.eq(mockTestCase); + expect(nock.isDone()).to.be.true; + }); + }); + + describe("batchUpsertTestCases", () => { + const mockTestCase = { displayName: "Case", aiInstructions: { steps: [] } }; + + it("should throw error if request fails", async () => { + nock(appDistributionOrigin()) + .post(`/v1alpha/${appName}/testCases:batchUpdate`) + .reply(400, { error: { status: "FAILED_PRECONDITION" } }); + await expect( + appDistributionClient.batchUpsertTestCases(appName, [mockTestCase]), + ).to.be.rejectedWith(FirebaseError, "Failed to upsert test cases"); + expect(nock.isDone()).to.be.true; + }); + + it("should resolve with TestCase when request succeeds", async () => { + nock(appDistributionOrigin()) + .post(`/v1alpha/${appName}/testCases:batchUpdate`) + .reply(200, { testCases: [mockTestCase] }); + await expect( + appDistributionClient.batchUpsertTestCases(appName, [mockTestCase]), + ).to.be.eventually.deep.eq([mockTestCase]); + expect(nock.isDone()).to.be.true; + }); + }); }); diff --git a/src/appdistribution/client.ts b/src/appdistribution/client.ts index 6cfde06de49..273033c2369 100644 --- a/src/appdistribution/client.ts +++ b/src/appdistribution/client.ts @@ -12,10 +12,14 @@ import { BatchRemoveTestersResponse, Group, ListGroupsResponse, + ListTestCasesResponse, ListTestersResponse, LoginCredential, mapDeviceToExecution, ReleaseTest, + TestCase, + BatchUpdateTestCasesRequest, + BatchUpdateTestCasesResponse, TestDevice, Tester, UploadReleaseResponse, @@ -295,4 +299,58 @@ export class AppDistributionClient { const response = await this.appDistroV1AlphaClient.get(releaseTestName); return response.body; } + + async listTestCases(appName: string): Promise { + const testCases: TestCase[] = []; + const client = this.appDistroV1AlphaClient; + + let pageToken: string | undefined; + do { + const queryParams: Record = pageToken ? { pageToken } : {}; + try { + const apiResponse = await client.get(`${appName}/testCases`, { + queryParams, + }); + testCases.push(...(apiResponse.body.testCases ?? [])); + pageToken = apiResponse.body.nextPageToken; + } catch (err) { + throw new FirebaseError(`Client failed to list test cases ${err}`); + } + } while (pageToken); + return testCases; + } + + async createTestCase(appName: string, testCase: TestCase): Promise { + try { + const response = await this.appDistroV1AlphaClient.request({ + method: "POST", + path: `${appName}/testCases`, + body: testCase, + }); + return response.body; + } catch (err: unknown) { + throw new FirebaseError(`Failed to create test case ${getErrMsg(err)}`); + } + } + + async batchUpsertTestCases(appName: string, testCases: TestCase[]): Promise { + try { + const response = await this.appDistroV1AlphaClient.request< + BatchUpdateTestCasesRequest, + BatchUpdateTestCasesResponse + >({ + method: "POST", + path: `${appName}/testCases:batchUpdate`, + body: { + requests: testCases.map((tc) => ({ + testCase: tc, + allowMissing: true, + })), + }, + }); + return response.body.testCases; + } catch (err: unknown) { + throw new FirebaseError(`Failed to upsert test cases ${getErrMsg(err)}`); + } + } } diff --git a/src/appdistribution/types.ts b/src/appdistribution/types.ts index f90155a4cd2..5fb7313b61f 100644 --- a/src/appdistribution/types.ts +++ b/src/appdistribution/types.ts @@ -128,3 +128,38 @@ export interface ReleaseTest { loginCredential?: LoginCredential; testCase?: string; } + +export interface AiStep { + goal: string; + hint?: string; + successCriteria?: string; +} + +export interface AiInstructions { + steps: AiStep[]; +} + +export interface TestCase { + name?: string; + displayName: string; + prerequisiteTestCase?: string; + aiInstructions: AiInstructions; +} + +export interface ListTestCasesResponse { + testCases: TestCase[]; + nextPageToken?: string; +} + +export interface UpdateTestCaseRequest { + testCase: TestCase; + allowMissing?: boolean; +} + +export interface BatchUpdateTestCasesRequest { + requests: UpdateTestCaseRequest[]; +} + +export interface BatchUpdateTestCasesResponse { + testCases: TestCase[]; +} diff --git a/src/appdistribution/yaml_helper.spec.ts b/src/appdistribution/yaml_helper.spec.ts new file mode 100644 index 00000000000..3a2b88f2c61 --- /dev/null +++ b/src/appdistribution/yaml_helper.spec.ts @@ -0,0 +1,119 @@ +import { TestCase } from "./types"; +import * as jsYaml from "js-yaml"; +import { fromYaml, toYaml } from "./yaml_helper"; +import { expect } from "chai"; + +const APP_NAME = "projects/12345/apps/1:12345:android:beef"; + +const TEST_CASE: TestCase = { + displayName: "test-display-name", + name: "projects/12345/apps/1:12345:android:beef/testCases/test-case-id", + prerequisiteTestCase: + "projects/12345/apps/1:12345:android:beef/testCases/prerequisite-test-case-id", + aiInstructions: { + steps: [ + { + goal: "test-goal", + hint: "test-hint", + successCriteria: "test-success-criteria", + }, + ], + }, +}; + +const YAML_STRING = `- displayName: test-display-name + id: test-case-id + prerequisiteTestCaseId: prerequisite-test-case-id + steps: + - goal: test-goal + hint: test-hint + successCriteria: test-success-criteria +`; + +const YAML_DATA = { + displayName: "test-display-name", + id: "test-case-id", + prerequisiteTestCaseId: "prerequisite-test-case-id", + steps: [ + { + goal: "test-goal", + hint: "test-hint", + successCriteria: "test-success-criteria", + }, + ], +}; + +describe("YamlHelper", () => { + it("converts TestCase[] to YAML string", () => { + const yamlString = toYaml([TEST_CASE]); + expect(yamlString).to.eq(YAML_STRING); // brittle ¯\_(ツ)_/¯ + expect(jsYaml.safeLoad(yamlString)).to.eql([YAML_DATA]); + }); + + it("converts YAML string to TestCase[]", () => { + const testCases = fromYaml(APP_NAME, YAML_STRING); + expect(testCases).to.eql([TEST_CASE]); + }); + + it("throws error if displayName is missing", () => { + expect(() => + fromYaml( + APP_NAME, + `- steps: + - goal: test-goal + hint: test-hint + successCriteria: test-success-criteria +`, + ), + ).to.throw(/"displayName" is required/); + }); + + it("throws error if steps is missing", () => { + expect(() => fromYaml(APP_NAME, `- displayName: test-display-name`)).to.throw( + /"steps" is required/, + ); + }); + + it("throws error if goal is missing", () => { + expect(() => + fromYaml( + APP_NAME, + `- displayName: test-display-name + steps: + - hint: test-hint + successCriteria: test-success-criteria +`, + ), + ).to.throw(/"goal" is required/); + }); + + it("throws error if additional property is present in test case", () => { + expect(() => + fromYaml( + APP_NAME, + `- displayName: test-display-name + extraTestCaseProperty: property + steps: + - goal: test-goal +`, + ), + ).to.throw(/unexpected property "extraTestCaseProperty"/); + }); + + it("throws error if additional property is present in step", () => { + expect(() => + fromYaml( + APP_NAME, + `- displayName: test-display-name + steps: + - goal: test-goal + extraStepProperty: property +`, + ), + ).to.throw(/unexpected property "extraStepProperty"/); + }); + + it("throws error if YAML is invalid", () => { + expect(() => fromYaml(APP_NAME, "this is not YAML")).to.throw(); + }); +}); diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts new file mode 100644 index 00000000000..6cfeb5825f7 --- /dev/null +++ b/src/appdistribution/yaml_helper.ts @@ -0,0 +1,94 @@ +import { TestCase } from "./types"; +import * as jsYaml from "js-yaml"; +import { FirebaseError } from "../error"; + +declare interface YamlStep { + goal?: string; + hint?: string; + successCriteria?: string; +} + +const ALLOWED_YAML_STEP_KEYS = new Set(["goal", "hint", "successCriteria"]); + +declare interface YamlTestCase { + displayName?: string; + id?: string; + prerequisiteTestCaseId?: string; + steps?: YamlStep[]; +} + +const ALLOWED_YAML_TEST_CASE_KEYS = new Set([ + "displayName", + "id", + "prerequisiteTestCaseId", + "steps", +]); + +function extractIdFromResourceName(name: string): string { + return name.split("/").pop() ?? ""; +} + +function toYamlTestCases(testCases: TestCase[]): YamlTestCase[] { + return testCases.map((testCase) => ({ + displayName: testCase.displayName ?? "", + id: extractIdFromResourceName(testCase.name!), + ...(testCase.prerequisiteTestCase && { + prerequisiteTestCaseId: extractIdFromResourceName(testCase.prerequisiteTestCase), + }), + steps: testCase.aiInstructions.steps.map((step) => ({ + goal: step.goal, + ...(step.hint && { hint: step.hint }), + ...(step.successCriteria && { successCriteria: step.successCriteria }), + })), + })); +} + +export function toYaml(testCases: TestCase[]): string { + return jsYaml.safeDump(toYamlTestCases(testCases)); +} + +function castExists(it: T | null | undefined, thing: string): T { + if (it == null) { + throw new FirebaseError(`"${thing}" is required`); + } + return it!; +} + +function checkAllowedKeys(allowedKeys: Set, o: object) { + for (const key of Object.keys(o)) { + if (!allowedKeys.has(key)) { + throw new FirebaseError(`unexpected property "${key}"`); + } + } +} + +function fromYamlTestCases(appName: string, yamlTestCases: YamlTestCase[]): TestCase[] { + return yamlTestCases.map((yamlTestCase) => { + checkAllowedKeys(ALLOWED_YAML_TEST_CASE_KEYS, yamlTestCase); + return { + displayName: castExists(yamlTestCase.displayName, "displayName"), + aiInstructions: { + steps: castExists(yamlTestCase.steps, "steps").map((yamlStep) => { + checkAllowedKeys(ALLOWED_YAML_STEP_KEYS, yamlStep); + return { + goal: castExists(yamlStep.goal, "goal"), + ...(yamlStep.hint && { hint: yamlStep.hint }), + ...(yamlStep.successCriteria && { + successCriteria: yamlStep.successCriteria, + }), + }; + }), + }, + ...(yamlTestCase.id && { + name: `${appName}/testCases/${yamlTestCase.id}`, + }), + ...(yamlTestCase.prerequisiteTestCaseId && { + prerequisiteTestCase: `${appName}/testCases/${yamlTestCase.prerequisiteTestCaseId}`, + }), + }; + }); +} + +export function fromYaml(appName: string, yaml: string): TestCase[] { + return fromYamlTestCases(appName, jsYaml.safeLoad(yaml) as YamlTestCase[]); +} diff --git a/src/commands/appdistribution-testcases-export.ts b/src/commands/appdistribution-testcases-export.ts new file mode 100644 index 00000000000..bfdc2aa0c6e --- /dev/null +++ b/src/commands/appdistribution-testcases-export.ts @@ -0,0 +1,30 @@ +import * as fs from "fs-extra"; +import { Command } from "../command"; +import { toYaml } from "../appdistribution/yaml_helper"; +import { requireAuth } from "../requireAuth"; +import { AppDistributionClient } from "../appdistribution/client"; +import { getAppName } from "../appdistribution/options-parser-util"; +import { TestCase, ListTestCasesResponse } from "../appdistribution/types"; +import { FirebaseError } from "../error"; +import * as utils from "../utils"; + +export const command = new Command("appdistribution:testcases:export ") + .description("export test cases as a YAML file") + .option("--app ", "the app id of your Firebase app") + .before(requireAuth) + .action(async (yamlFile: string, options?: any): Promise => { + const appName = getAppName(options); + const appDistroClient = new AppDistributionClient(); + let testCases: TestCase[]; + try { + testCases = await appDistroClient.listTestCases(appName); + } catch (err: any) { + throw new FirebaseError("Failed to list test cases.", { + exit: 1, + original: err, + }); + } + fs.writeFileSync(yamlFile, toYaml(testCases), "utf8"); + utils.logSuccess(`Exported ${testCases.length} test cases to ${yamlFile}`); + return { testCases }; + }); diff --git a/src/commands/appdistribution-testcases-import.ts b/src/commands/appdistribution-testcases-import.ts new file mode 100644 index 00000000000..90693f2339b --- /dev/null +++ b/src/commands/appdistribution-testcases-import.ts @@ -0,0 +1,30 @@ +import * as fs from "fs-extra"; +import { Command } from "../command"; +import { fromYaml } from "../appdistribution/yaml_helper"; +import { requireAuth } from "../requireAuth"; +import { AppDistributionClient } from "../appdistribution/client"; +import { getAppName, ensureFileExists } from "../appdistribution/options-parser-util"; +import { TestCase } from "../appdistribution/types"; +import * as utils from "../utils"; + +export const command = new Command("appdistribution:testcases:import ") + .description("import test cases from YAML file") + .option("--app ", "the app id of your Firebase app") + .before(requireAuth) + .action(async (yamlFile: string, options?: any) => { + const appName = getAppName(options); + const appDistroClient = new AppDistributionClient(); + ensureFileExists(yamlFile); + const testCases: TestCase[] = fromYaml(appName, fs.readFileSync(yamlFile, "utf8")); + const testCasesWithoutName = testCases.filter((tc) => !tc.name); + // nothing can depend on these test cases yet, since they don't have + // resource names + await Promise.all( + testCasesWithoutName.map((tc) => appDistroClient.createTestCase(appName, tc)), + ); + // any test case with a resource name can be referenced by any other new or + // existing test case, so we need to batch the upsert. + const testCasesWithName = testCases.filter((tc) => !!tc.name); + await appDistroClient.batchUpsertTestCases(appName, testCasesWithName); + utils.logSuccess(`Imported ${testCases.length} test cases from ${yamlFile}`); + }); diff --git a/src/commands/index.ts b/src/commands/index.ts index e18f1aa23f8..1a550749e12 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -30,6 +30,9 @@ export function load(client: any): any { client.appdistribution.groups.create = loadCommand("appdistribution-groups-create"); client.appdistribution.groups.delete = loadCommand("appdistribution-groups-delete"); client.appdistribution.group = client.appdistribution.groups; + client.appdistribution.testCases = {}; + client.appdistribution.testCases.export = loadCommand("appdistribution-testcases-export"); + client.appdistribution.testCases.import = loadCommand("appdistribution-testcases-import"); client.apps = {}; client.apps.create = loadCommand("apps-create"); client.apps.list = loadCommand("apps-list"); From e2a8ae04c70bfe3de02e72086e2076e1c158790e Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Tue, 28 Oct 2025 18:23:01 -0400 Subject: [PATCH 02/12] add explicit `unknown` type to caught errors --- src/appdistribution/client.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/appdistribution/client.ts b/src/appdistribution/client.ts index 273033c2369..43ac9a3ae75 100644 --- a/src/appdistribution/client.ts +++ b/src/appdistribution/client.ts @@ -148,7 +148,7 @@ export class AppDistributionClient { apiResponse = await client.get(`${projectName}/testers`, { queryParams, }); - } catch (err) { + } catch (err: unknown) { throw new FirebaseError(`Client request failed to list testers ${err}`); } @@ -210,7 +210,7 @@ export class AppDistributionClient { }); groups.push(...(apiResponse.body.groups ?? [])); pageToken = apiResponse.body.nextPageToken; - } catch (err) { + } catch (err: unknown) { throw new FirebaseError(`Client failed to list groups ${err}`); } } while (pageToken); @@ -313,7 +313,7 @@ export class AppDistributionClient { }); testCases.push(...(apiResponse.body.testCases ?? [])); pageToken = apiResponse.body.nextPageToken; - } catch (err) { + } catch (err: unknown) { throw new FirebaseError(`Client failed to list test cases ${err}`); } } while (pageToken); From b4a68f5bdd4d29c9bc5055a66ce0b2a18d9dccd3 Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Tue, 28 Oct 2025 18:23:36 -0400 Subject: [PATCH 03/12] better tests --- src/appdistribution/yaml_helper.spec.ts | 46 +++++++++++++++++-------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/src/appdistribution/yaml_helper.spec.ts b/src/appdistribution/yaml_helper.spec.ts index 3a2b88f2c61..8ff06924ad7 100644 --- a/src/appdistribution/yaml_helper.spec.ts +++ b/src/appdistribution/yaml_helper.spec.ts @@ -5,21 +5,27 @@ import { expect } from "chai"; const APP_NAME = "projects/12345/apps/1:12345:android:beef"; -const TEST_CASE: TestCase = { - displayName: "test-display-name", - name: "projects/12345/apps/1:12345:android:beef/testCases/test-case-id", - prerequisiteTestCase: - "projects/12345/apps/1:12345:android:beef/testCases/prerequisite-test-case-id", - aiInstructions: { - steps: [ - { - goal: "test-goal", - hint: "test-hint", - successCriteria: "test-success-criteria", - }, - ], +const TEST_CASES: TestCase[] = [ + { + displayName: "test-display-name", + name: "projects/12345/apps/1:12345:android:beef/testCases/test-case-id", + prerequisiteTestCase: + "projects/12345/apps/1:12345:android:beef/testCases/prerequisite-test-case-id", + aiInstructions: { + steps: [ + { + goal: "test-goal", + hint: "test-hint", + successCriteria: "test-success-criteria", + }, + ], + }, }, -}; + { + displayName: "minimal-case", + aiInstructions: { steps: [{ goal: "test-goal" }] }, + }, +]; const YAML_STRING = `- displayName: test-display-name id: test-case-id @@ -114,6 +120,16 @@ describe("YamlHelper", () => { }); it("throws error if YAML is invalid", () => { - expect(() => fromYaml(APP_NAME, "this is not YAML")).to.throw(); + expect(() => + fromYaml( + APP_NAME, + `- +this is not valid YAML`, + ), + ).to.throw(/at line 2/); + }); + + it("throws error if YAML doesn't conatin a top-level array", () => { + expect(() => fromYaml(APP_NAME, "not a list")).to.throw(/must contain a list of test cases/); }); }); From 28245e32d77138fcb7838e900884e2b311ae72b8 Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Tue, 28 Oct 2025 18:34:22 -0400 Subject: [PATCH 04/12] Better YAML errors and test coverage --- src/appdistribution/yaml_helper.spec.ts | 62 ++++++++++++++++++------- src/appdistribution/yaml_helper.ts | 5 +- 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/src/appdistribution/yaml_helper.spec.ts b/src/appdistribution/yaml_helper.spec.ts index 8ff06924ad7..a299a45d0dc 100644 --- a/src/appdistribution/yaml_helper.spec.ts +++ b/src/appdistribution/yaml_helper.spec.ts @@ -1,5 +1,5 @@ -import { TestCase } from "./types"; import * as jsYaml from "js-yaml"; +import { TestCase } from "./types"; import { fromYaml, toYaml } from "./yaml_helper"; import { expect } from "chai"; @@ -23,7 +23,8 @@ const TEST_CASES: TestCase[] = [ }, { displayName: "minimal-case", - aiInstructions: { steps: [{ goal: "test-goal" }] }, + name: "projects/12345/apps/1:12345:android:beef/testCases/minimal-id", + aiInstructions: { steps: [{ goal: "win" }] }, }, ]; @@ -34,31 +35,58 @@ const YAML_STRING = `- displayName: test-display-name - goal: test-goal hint: test-hint successCriteria: test-success-criteria +- displayName: minimal-case + id: minimal-id + steps: + - goal: win `; -const YAML_DATA = { - displayName: "test-display-name", - id: "test-case-id", - prerequisiteTestCaseId: "prerequisite-test-case-id", - steps: [ - { - goal: "test-goal", - hint: "test-hint", - successCriteria: "test-success-criteria", - }, - ], -}; +const YAML_DATA = [ + { + displayName: "test-display-name", + id: "test-case-id", + prerequisiteTestCaseId: "prerequisite-test-case-id", + steps: [ + { + goal: "test-goal", + hint: "test-hint", + successCriteria: "test-success-criteria", + }, + ], + }, + { + displayName: "minimal-case", + id: "minimal-id", + steps: [{ goal: "win" }], + }, +]; describe("YamlHelper", () => { it("converts TestCase[] to YAML string", () => { - const yamlString = toYaml([TEST_CASE]); + const yamlString = toYaml(TEST_CASES); + expect(jsYaml.safeLoad(yamlString)).to.eql(YAML_DATA); expect(yamlString).to.eq(YAML_STRING); // brittle ¯\_(ツ)_/¯ - expect(jsYaml.safeLoad(yamlString)).to.eql([YAML_DATA]); }); it("converts YAML string to TestCase[]", () => { const testCases = fromYaml(APP_NAME, YAML_STRING); - expect(testCases).to.eql([TEST_CASE]); + expect(testCases).to.eql(TEST_CASES); + }); + + it("converts YAML without ID", () => { + const testCases = fromYaml( + APP_NAME, + `- displayName: minimal-case + steps: + - goal: win +`, + ); + expect(testCases).to.eql([ + { + displayName: "minimal-case", + aiInstructions: { steps: [{ goal: "win" }] }, + }, + ]); }); it("throws error if displayName is missing", () => { diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts index 6cfeb5825f7..af50d20b522 100644 --- a/src/appdistribution/yaml_helper.ts +++ b/src/appdistribution/yaml_helper.ts @@ -1,6 +1,6 @@ -import { TestCase } from "./types"; import * as jsYaml from "js-yaml"; import { FirebaseError } from "../error"; +import { TestCase } from "./types"; declare interface YamlStep { goal?: string; @@ -63,6 +63,9 @@ function checkAllowedKeys(allowedKeys: Set, o: object) { } function fromYamlTestCases(appName: string, yamlTestCases: YamlTestCase[]): TestCase[] { + if (!Array.isArray(yamlTestCases)) { + throw new FirebaseError("YAML file must contain a list of test cases."); + } return yamlTestCases.map((yamlTestCase) => { checkAllowedKeys(ALLOWED_YAML_TEST_CASE_KEYS, yamlTestCase); return { From dd87b4e640752cca1701172215a2472cbedcbfea Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Tue, 28 Oct 2025 18:42:01 -0400 Subject: [PATCH 05/12] inline simplified `mapDeviceToExecution()`. --- src/appdistribution/client.ts | 7 +++---- src/appdistribution/types.ts | 11 ----------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/appdistribution/client.ts b/src/appdistribution/client.ts index 43ac9a3ae75..50da3cf5a25 100644 --- a/src/appdistribution/client.ts +++ b/src/appdistribution/client.ts @@ -10,16 +10,15 @@ import { appDistributionOrigin } from "../api"; import { AabInfo, BatchRemoveTestersResponse, + BatchUpdateTestCasesRequest, + BatchUpdateTestCasesResponse, Group, ListGroupsResponse, ListTestCasesResponse, ListTestersResponse, LoginCredential, - mapDeviceToExecution, ReleaseTest, TestCase, - BatchUpdateTestCasesRequest, - BatchUpdateTestCasesResponse, TestDevice, Tester, UploadReleaseResponse, @@ -284,7 +283,7 @@ export class AppDistributionClient { method: "POST", path: `${releaseName}/tests`, body: { - deviceExecutions: devices.map(mapDeviceToExecution), + deviceExecutions: devices.map((device) => ({device})), loginCredential, testCase: testCaseName, }, diff --git a/src/appdistribution/types.ts b/src/appdistribution/types.ts index 5fb7313b61f..4c442d1c928 100644 --- a/src/appdistribution/types.ts +++ b/src/appdistribution/types.ts @@ -100,17 +100,6 @@ export interface DeviceExecution { inconclusiveReason?: string; } -export function mapDeviceToExecution(device: TestDevice): DeviceExecution { - return { - device: { - model: device.model, - version: device.version, - orientation: device.orientation, - locale: device.locale, - }, - }; -} - export interface FieldHints { usernameResourceName?: string; passwordResourceName?: string; From 888fee54823e6b40d0198383686be540c73b60db Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Tue, 28 Oct 2025 18:53:42 -0400 Subject: [PATCH 06/12] Even better YAML errors --- src/appdistribution/yaml_helper.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts index af50d20b522..ffe8ccbfccb 100644 --- a/src/appdistribution/yaml_helper.ts +++ b/src/appdistribution/yaml_helper.ts @@ -1,5 +1,5 @@ import * as jsYaml from "js-yaml"; -import { FirebaseError } from "../error"; +import { getErrMsg, FirebaseError } from "../error"; import { TestCase } from "./types"; declare interface YamlStep { @@ -63,9 +63,7 @@ function checkAllowedKeys(allowedKeys: Set, o: object) { } function fromYamlTestCases(appName: string, yamlTestCases: YamlTestCase[]): TestCase[] { - if (!Array.isArray(yamlTestCases)) { - throw new FirebaseError("YAML file must contain a list of test cases."); - } + return yamlTestCases.map((yamlTestCase) => { checkAllowedKeys(ALLOWED_YAML_TEST_CASE_KEYS, yamlTestCase); return { @@ -93,5 +91,14 @@ function fromYamlTestCases(appName: string, yamlTestCases: YamlTestCase[]): Test } export function fromYaml(appName: string, yaml: string): TestCase[] { - return fromYamlTestCases(appName, jsYaml.safeLoad(yaml) as YamlTestCase[]); + let parsedYaml: unknown + try { + parsedYaml = jsYaml.safeLoad(yaml) + } catch (err: unknown) { + throw new FirebaseError(`Failed to parse YAML: ${getErrMsg(err)}`) + } + if (!Array.isArray(parsedYaml)) { + throw new FirebaseError("YAML file must contain a list of test cases."); + } + return fromYamlTestCases(appName, parsedYaml as YamlTestCase[]); } From cfc5cf440db8af081af67a5c87652fe854e7a7a2 Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Tue, 28 Oct 2025 19:00:25 -0400 Subject: [PATCH 07/12] fix lint --- src/appdistribution/client.ts | 2 +- src/appdistribution/yaml_helper.ts | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/appdistribution/client.ts b/src/appdistribution/client.ts index 50da3cf5a25..e848827df7d 100644 --- a/src/appdistribution/client.ts +++ b/src/appdistribution/client.ts @@ -283,7 +283,7 @@ export class AppDistributionClient { method: "POST", path: `${releaseName}/tests`, body: { - deviceExecutions: devices.map((device) => ({device})), + deviceExecutions: devices.map((device) => ({ device })), loginCredential, testCase: testCaseName, }, diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts index ffe8ccbfccb..8e81a2d9cd7 100644 --- a/src/appdistribution/yaml_helper.ts +++ b/src/appdistribution/yaml_helper.ts @@ -63,7 +63,6 @@ function checkAllowedKeys(allowedKeys: Set, o: object) { } function fromYamlTestCases(appName: string, yamlTestCases: YamlTestCase[]): TestCase[] { - return yamlTestCases.map((yamlTestCase) => { checkAllowedKeys(ALLOWED_YAML_TEST_CASE_KEYS, yamlTestCase); return { @@ -91,11 +90,11 @@ function fromYamlTestCases(appName: string, yamlTestCases: YamlTestCase[]): Test } export function fromYaml(appName: string, yaml: string): TestCase[] { - let parsedYaml: unknown + let parsedYaml: unknown; try { - parsedYaml = jsYaml.safeLoad(yaml) + parsedYaml = jsYaml.safeLoad(yaml); } catch (err: unknown) { - throw new FirebaseError(`Failed to parse YAML: ${getErrMsg(err)}`) + throw new FirebaseError(`Failed to parse YAML: ${getErrMsg(err)}`); } if (!Array.isArray(parsedYaml)) { throw new FirebaseError("YAML file must contain a list of test cases."); From dd428dd09df990fedfa734cb913dd2ed2bd1d71d Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Thu, 30 Oct 2025 00:19:59 -0400 Subject: [PATCH 08/12] address feedback --- src/appdistribution/client.spec.ts | 4 ++-- src/appdistribution/client.ts | 8 ++++---- src/appdistribution/yaml_helper.spec.ts | 2 +- src/appdistribution/yaml_helper.ts | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/appdistribution/client.spec.ts b/src/appdistribution/client.spec.ts index c95f1cf91d0..81e9e050853 100644 --- a/src/appdistribution/client.spec.ts +++ b/src/appdistribution/client.spec.ts @@ -517,7 +517,7 @@ describe("distribution", () => { it("should resolve with array of test cases when request succeeds", async () => { const testCases: TestCase[] = [ { - name: `$appName/testCases/tc_1`, + name: `${appName}/testCases/tc_1`, displayName: "Test Case 1", aiInstructions: { steps: [ @@ -528,7 +528,7 @@ describe("distribution", () => { }, }, { - name: `$appName/testCases/tc_2`, + name: `${appName}/testCases/tc_2`, displayName: "Test Case 2", aiInstructions: { steps: [] }, }, diff --git a/src/appdistribution/client.ts b/src/appdistribution/client.ts index e848827df7d..a286beebd0d 100644 --- a/src/appdistribution/client.ts +++ b/src/appdistribution/client.ts @@ -115,7 +115,7 @@ export class AppDistributionClient { try { await this.appDistroV1Client.post(`/${releaseName}:distribute`, data); } catch (err: any) { - let errorMessage = err.message; + let errorMessage = getErrMsg(err); const errorStatus = err?.context?.body?.error?.status; if (errorStatus === "FAILED_PRECONDITION") { errorMessage = "invalid testers"; @@ -148,7 +148,7 @@ export class AppDistributionClient { queryParams, }); } catch (err: unknown) { - throw new FirebaseError(`Client request failed to list testers ${err}`); + throw new FirebaseError(`Client request failed to list testers ${getErrMsg(err)}`); } for (const t of apiResponse.body.testers ?? []) { @@ -210,7 +210,7 @@ export class AppDistributionClient { groups.push(...(apiResponse.body.groups ?? [])); pageToken = apiResponse.body.nextPageToken; } catch (err: unknown) { - throw new FirebaseError(`Client failed to list groups ${err}`); + throw new FirebaseError(`Client failed to list groups ${getErrMsg(err)}`); } } while (pageToken); return groups; @@ -313,7 +313,7 @@ export class AppDistributionClient { testCases.push(...(apiResponse.body.testCases ?? [])); pageToken = apiResponse.body.nextPageToken; } catch (err: unknown) { - throw new FirebaseError(`Client failed to list test cases ${err}`); + throw new FirebaseError(`Client failed to list test cases ${getErrMsg(err)}`); } } while (pageToken); return testCases; diff --git a/src/appdistribution/yaml_helper.spec.ts b/src/appdistribution/yaml_helper.spec.ts index a299a45d0dc..a881a7c8e2f 100644 --- a/src/appdistribution/yaml_helper.spec.ts +++ b/src/appdistribution/yaml_helper.spec.ts @@ -157,7 +157,7 @@ this is not valid YAML`, ).to.throw(/at line 2/); }); - it("throws error if YAML doesn't conatin a top-level array", () => { + it("throws error if YAML doesn't contain a top-level array", () => { expect(() => fromYaml(APP_NAME, "not a list")).to.throw(/must contain a list of test cases/); }); }); diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts index 8e81a2d9cd7..466467dafdc 100644 --- a/src/appdistribution/yaml_helper.ts +++ b/src/appdistribution/yaml_helper.ts @@ -30,8 +30,8 @@ function extractIdFromResourceName(name: string): string { function toYamlTestCases(testCases: TestCase[]): YamlTestCase[] { return testCases.map((testCase) => ({ - displayName: testCase.displayName ?? "", - id: extractIdFromResourceName(testCase.name!), + displayName: testCase.displayName, + id: extractIdFromResourceName(testCase.name!), // resource name is retured by server ...(testCase.prerequisiteTestCase && { prerequisiteTestCaseId: extractIdFromResourceName(testCase.prerequisiteTestCase), }), From c02a4f5a596203b1bd7975f4057bcc2577231bb4 Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Thu, 30 Oct 2025 12:27:40 -0400 Subject: [PATCH 09/12] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ee3d69a119..a2e65533803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ - Added `functions.list_functions` as a MCP tool (#9369) - Added AI Logic to `firebase init` CLI command and `firebase_init` MCP tool. (#9185) - Improved error messages for Firebase AI Logic provisioning during 'firebase init' (#9377) +- Added `appdistribution:testcases:export` and `appdistribution:testcases:import` (#9397) From 794a76f8facbb9e9d2381085340068cc8b61f70d Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Thu, 30 Oct 2025 13:19:03 -0400 Subject: [PATCH 10/12] Add missing App Distribution commands --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 06e65e51f2e..cf4162082e0 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,17 @@ These commands let you deploy and interact with your Firebase services. ### App Distribution Commands -| Command | Description | -| ------------------------------ | ---------------------- | -| **appdistribution:distribute** | Upload a distribution. | +| Command | Description | +| ------------------------------------ | ---------------------------------------------------------------------------------------- | +| **appdistribution:distribute** | Upload a release binary and optionally distribute it to testers and run automated tests. | +| **appdistribution:testers:list** | List testers in project. | +| **appdistribution:testers:add** | Add testers to project (and group, if specified via flag). | +| **appdistribution:testers:remove** | Remove testers from a project (or group, if specified via flag). | +| **appdistribution:groups:list** | List groups (of testers). | +| **appdistribution:groups:create** | Create a group (of testers). | +| **appdistribution:groups:delete** | Delete a group (of testers). | +| **appdistribution:testcases:export** | Export test cases as a YAML file. | +| **appdistribution:testcases:import** | Import test cases from YAML file. | ### Auth Commands From ee7f2386b821aa2dc06750d43c9248d46b7caa44 Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Thu, 30 Oct 2025 13:25:48 -0400 Subject: [PATCH 11/12] prettier --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cf4162082e0..3facfc94e28 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ These commands let you deploy and interact with your Firebase services. | Command | Description | | ------------------------------------ | ---------------------------------------------------------------------------------------- | -| **appdistribution:distribute** | Upload a release binary and optionally distribute it to testers and run automated tests. | +| **appdistribution:distribute** | Upload a release binary and optionally distribute it to testers and run automated tests. | | **appdistribution:testers:list** | List testers in project. | | **appdistribution:testers:add** | Add testers to project (and group, if specified via flag). | | **appdistribution:testers:remove** | Remove testers from a project (or group, if specified via flag). | From 66806108e60eba904437df96f37dcfef5bc884b9 Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Thu, 30 Oct 2025 15:22:53 -0400 Subject: [PATCH 12/12] Better error message for partial create success --- src/commands/appdistribution-testcases-import.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/commands/appdistribution-testcases-import.ts b/src/commands/appdistribution-testcases-import.ts index 90693f2339b..343ef25ee11 100644 --- a/src/commands/appdistribution-testcases-import.ts +++ b/src/commands/appdistribution-testcases-import.ts @@ -6,6 +6,7 @@ import { AppDistributionClient } from "../appdistribution/client"; import { getAppName, ensureFileExists } from "../appdistribution/options-parser-util"; import { TestCase } from "../appdistribution/types"; import * as utils from "../utils"; +import { FirebaseError } from "../error"; export const command = new Command("appdistribution:testcases:import ") .description("import test cases from YAML file") @@ -19,9 +20,19 @@ export const command = new Command("appdistribution:testcases:import !tc.name); // nothing can depend on these test cases yet, since they don't have // resource names - await Promise.all( + const creationResults = await Promise.allSettled( testCasesWithoutName.map((tc) => appDistroClient.createTestCase(appName, tc)), ); + const failed = creationResults.filter((r) => r.status === "rejected"); + if (failed.length > 0) { + for (const f of failed) { + utils.logWarning((f as any).reason); + } + const succeeded = creationResults.length - failed.length; + throw new FirebaseError( + `Created ${succeeded} test case(s), but failed to create ${failed.length}.`, + ); + } // any test case with a resource name can be referenced by any other new or // existing test case, so we need to batch the upsert. const testCasesWithName = testCases.filter((tc) => !!tc.name);