diff --git a/CHANGELOG.md b/CHANGELOG.md index 76763e39c62..c74c196543f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ - Added `functions.list_functions` as a MCP tool (#9369) - Added AI Logic to `firebase init` CLI command and `firebase_init` MCP tool. (#9185) - Improved error messages for Firebase AI Logic provisioning during 'firebase init' (#9377) +- Added `appdistribution:testcases:export` and `appdistribution:testcases:import` (#9397) - Updated to v2.16.0 of the Data Connect emulator, which includes internal improvements. - Data Connect now allows executing a valid query / operation even if the other operations are invalid. (This toleration provides convenience on a best-effort basis. Some errors like invalid syntax can still cause the whole request to be rejected.) - Fixed enum list deserialization in Data Connect generated Dart SDKs. diff --git a/README.md b/README.md index 06e65e51f2e..3facfc94e28 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,17 @@ These commands let you deploy and interact with your Firebase services. ### App Distribution Commands -| Command | Description | -| ------------------------------ | ---------------------- | -| **appdistribution:distribute** | Upload a distribution. | +| Command | Description | +| ------------------------------------ | ---------------------------------------------------------------------------------------- | +| **appdistribution:distribute** | Upload a release binary and optionally distribute it to testers and run automated tests. | +| **appdistribution:testers:list** | List testers in project. | +| **appdistribution:testers:add** | Add testers to project (and group, if specified via flag). | +| **appdistribution:testers:remove** | Remove testers from a project (or group, if specified via flag). | +| **appdistribution:groups:list** | List groups (of testers). | +| **appdistribution:groups:create** | Create a group (of testers). | +| **appdistribution:groups:delete** | Delete a group (of testers). | +| **appdistribution:testcases:export** | Export test cases as a YAML file. | +| **appdistribution:testcases:import** | Import test cases from YAML file. | ### Auth Commands diff --git a/package.json b/package.json index 2539ebfd5a6..2795e35480a 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "mocha": "nyc --reporter=html mocha 'src/**/*.spec.{ts,js}'", "prepare": "npm run clean && npm run build:publish", "test": "npm run lint:quiet && npm run test:compile && npm run mocha", + "test:appdistribution": "npm run lint:quiet && npm run test:compile && nyc mocha 'src/appdistribution/*.spec.{ts,js}'", "test:apptesting": "npm run lint:quiet && npm run test:compile && nyc mocha 'src/apptesting/*.spec.{ts,js}'", "test:client-integration": "bash ./scripts/client-integration-tests/run.sh", "test:compile": "tsc --project tsconfig.compile.json", diff --git a/src/appdistribution/client.spec.ts b/src/appdistribution/client.spec.ts index 8a24a84161f..81e9e050853 100644 --- a/src/appdistribution/client.spec.ts +++ b/src/appdistribution/client.spec.ts @@ -7,7 +7,7 @@ import * as sinon from "sinon"; import * as tmp from "tmp"; import { AppDistributionClient } from "./client"; -import { BatchRemoveTestersResponse, Group, TestDevice } from "./types"; +import { BatchRemoveTestersResponse, Group, TestCase, TestDevice } from "./types"; import { appDistributionOrigin } from "../api"; import { Distribution } from "./distribution"; import { FirebaseError } from "../error"; @@ -501,4 +501,91 @@ describe("distribution", () => { expect(nock.isDone()).to.be.true; }); }); + + describe("listTestCases", () => { + it("should throw error if request fails", async () => { + nock(appDistributionOrigin()) + .get(`/v1alpha/${appName}/testCases`) + .reply(400, { error: { status: "FAILED_PRECONDITION" } }); + await expect(appDistributionClient.listTestCases(appName)).to.be.rejectedWith( + FirebaseError, + "Client failed to list test cases", + ); + expect(nock.isDone()).to.be.true; + }); + + it("should resolve with array of test cases when request succeeds", async () => { + const testCases: TestCase[] = [ + { + name: `${appName}/testCases/tc_1`, + displayName: "Test Case 1", + aiInstructions: { + steps: [ + { + goal: "Win at all costs", + }, + ], + }, + }, + { + name: `${appName}/testCases/tc_2`, + displayName: "Test Case 2", + aiInstructions: { steps: [] }, + }, + ]; + + nock(appDistributionOrigin()).get(`/v1alpha/${appName}/testCases`).reply(200, { + testCases: testCases, + }); + await expect(appDistributionClient.listTestCases(appName)).to.eventually.deep.eq(testCases); + expect(nock.isDone()).to.be.true; + }); + }); + + describe("createTestCase", () => { + const mockTestCase = { displayName: "Case", aiInstructions: { steps: [] } }; + + it("should throw error if request fails", async () => { + nock(appDistributionOrigin()) + .post(`/v1alpha/${appName}/testCases`) + .reply(400, { error: { status: "FAILED_PRECONDITION" } }); + await expect(appDistributionClient.createTestCase(appName, mockTestCase)).to.be.rejectedWith( + FirebaseError, + "Failed to create test case", + ); + expect(nock.isDone()).to.be.true; + }); + + it("should resolve with TestCase when request succeeds", async () => { + nock(appDistributionOrigin()).post(`/v1alpha/${appName}/testCases`).reply(200, mockTestCase); + await expect( + appDistributionClient.createTestCase(appName, mockTestCase), + ).to.be.eventually.deep.eq(mockTestCase); + expect(nock.isDone()).to.be.true; + }); + }); + + describe("batchUpsertTestCases", () => { + const mockTestCase = { displayName: "Case", aiInstructions: { steps: [] } }; + + it("should throw error if request fails", async () => { + nock(appDistributionOrigin()) + .post(`/v1alpha/${appName}/testCases:batchUpdate`) + .reply(400, { error: { status: "FAILED_PRECONDITION" } }); + await expect( + appDistributionClient.batchUpsertTestCases(appName, [mockTestCase]), + ).to.be.rejectedWith(FirebaseError, "Failed to upsert test cases"); + expect(nock.isDone()).to.be.true; + }); + + it("should resolve with TestCase when request succeeds", async () => { + nock(appDistributionOrigin()) + .post(`/v1alpha/${appName}/testCases:batchUpdate`) + .reply(200, { testCases: [mockTestCase] }); + await expect( + appDistributionClient.batchUpsertTestCases(appName, [mockTestCase]), + ).to.be.eventually.deep.eq([mockTestCase]); + expect(nock.isDone()).to.be.true; + }); + }); }); diff --git a/src/appdistribution/client.ts b/src/appdistribution/client.ts index 6cfde06de49..a286beebd0d 100644 --- a/src/appdistribution/client.ts +++ b/src/appdistribution/client.ts @@ -10,12 +10,15 @@ import { appDistributionOrigin } from "../api"; import { AabInfo, BatchRemoveTestersResponse, + BatchUpdateTestCasesRequest, + BatchUpdateTestCasesResponse, Group, ListGroupsResponse, + ListTestCasesResponse, ListTestersResponse, LoginCredential, - mapDeviceToExecution, ReleaseTest, + TestCase, TestDevice, Tester, UploadReleaseResponse, @@ -112,7 +115,7 @@ export class AppDistributionClient { try { await this.appDistroV1Client.post(`/${releaseName}:distribute`, data); } catch (err: any) { - let errorMessage = err.message; + let errorMessage = getErrMsg(err); const errorStatus = err?.context?.body?.error?.status; if (errorStatus === "FAILED_PRECONDITION") { errorMessage = "invalid testers"; @@ -144,8 +147,8 @@ export class AppDistributionClient { apiResponse = await client.get(`${projectName}/testers`, { queryParams, }); - } catch (err) { - throw new FirebaseError(`Client request failed to list testers ${err}`); + } catch (err: unknown) { + throw new FirebaseError(`Client request failed to list testers ${getErrMsg(err)}`); } for (const t of apiResponse.body.testers ?? []) { @@ -206,8 +209,8 @@ export class AppDistributionClient { }); groups.push(...(apiResponse.body.groups ?? [])); pageToken = apiResponse.body.nextPageToken; - } catch (err) { - throw new FirebaseError(`Client failed to list groups ${err}`); + } catch (err: unknown) { + throw new FirebaseError(`Client failed to list groups ${getErrMsg(err)}`); } } while (pageToken); return groups; @@ -280,7 +283,7 @@ export class AppDistributionClient { method: "POST", path: `${releaseName}/tests`, body: { - deviceExecutions: devices.map(mapDeviceToExecution), + deviceExecutions: devices.map((device) => ({ device })), loginCredential, testCase: testCaseName, }, @@ -295,4 +298,58 @@ export class AppDistributionClient { const response = await this.appDistroV1AlphaClient.get(releaseTestName); return response.body; } + + async listTestCases(appName: string): Promise { + const testCases: TestCase[] = []; + const client = this.appDistroV1AlphaClient; + + let pageToken: string | undefined; + do { + const queryParams: Record = pageToken ? { pageToken } : {}; + try { + const apiResponse = await client.get(`${appName}/testCases`, { + queryParams, + }); + testCases.push(...(apiResponse.body.testCases ?? [])); + pageToken = apiResponse.body.nextPageToken; + } catch (err: unknown) { + throw new FirebaseError(`Client failed to list test cases ${getErrMsg(err)}`); + } + } while (pageToken); + return testCases; + } + + async createTestCase(appName: string, testCase: TestCase): Promise { + try { + const response = await this.appDistroV1AlphaClient.request({ + method: "POST", + path: `${appName}/testCases`, + body: testCase, + }); + return response.body; + } catch (err: unknown) { + throw new FirebaseError(`Failed to create test case ${getErrMsg(err)}`); + } + } + + async batchUpsertTestCases(appName: string, testCases: TestCase[]): Promise { + try { + const response = await this.appDistroV1AlphaClient.request< + BatchUpdateTestCasesRequest, + BatchUpdateTestCasesResponse + >({ + method: "POST", + path: `${appName}/testCases:batchUpdate`, + body: { + requests: testCases.map((tc) => ({ + testCase: tc, + allowMissing: true, + })), + }, + }); + return response.body.testCases; + } catch (err: unknown) { + throw new FirebaseError(`Failed to upsert test cases ${getErrMsg(err)}`); + } + } } diff --git a/src/appdistribution/types.ts b/src/appdistribution/types.ts index f90155a4cd2..4c442d1c928 100644 --- a/src/appdistribution/types.ts +++ b/src/appdistribution/types.ts @@ -100,17 +100,6 @@ export interface DeviceExecution { inconclusiveReason?: string; } -export function mapDeviceToExecution(device: TestDevice): DeviceExecution { - return { - device: { - model: device.model, - version: device.version, - orientation: device.orientation, - locale: device.locale, - }, - }; -} - export interface FieldHints { usernameResourceName?: string; passwordResourceName?: string; @@ -128,3 +117,38 @@ export interface ReleaseTest { loginCredential?: LoginCredential; testCase?: string; } + +export interface AiStep { + goal: string; + hint?: string; + successCriteria?: string; +} + +export interface AiInstructions { + steps: AiStep[]; +} + +export interface TestCase { + name?: string; + displayName: string; + prerequisiteTestCase?: string; + aiInstructions: AiInstructions; +} + +export interface ListTestCasesResponse { + testCases: TestCase[]; + nextPageToken?: string; +} + +export interface UpdateTestCaseRequest { + testCase: TestCase; + allowMissing?: boolean; +} + +export interface BatchUpdateTestCasesRequest { + requests: UpdateTestCaseRequest[]; +} + +export interface BatchUpdateTestCasesResponse { + testCases: TestCase[]; +} diff --git a/src/appdistribution/yaml_helper.spec.ts b/src/appdistribution/yaml_helper.spec.ts new file mode 100644 index 00000000000..a881a7c8e2f --- /dev/null +++ b/src/appdistribution/yaml_helper.spec.ts @@ -0,0 +1,163 @@ +import * as jsYaml from "js-yaml"; +import { TestCase } from "./types"; +import { fromYaml, toYaml } from "./yaml_helper"; +import { expect } from "chai"; + +const APP_NAME = "projects/12345/apps/1:12345:android:beef"; + +const TEST_CASES: TestCase[] = [ + { + displayName: "test-display-name", + name: "projects/12345/apps/1:12345:android:beef/testCases/test-case-id", + prerequisiteTestCase: + "projects/12345/apps/1:12345:android:beef/testCases/prerequisite-test-case-id", + aiInstructions: { + steps: [ + { + goal: "test-goal", + hint: "test-hint", + successCriteria: "test-success-criteria", + }, + ], + }, + }, + { + displayName: "minimal-case", + name: "projects/12345/apps/1:12345:android:beef/testCases/minimal-id", + aiInstructions: { steps: [{ goal: "win" }] }, + }, +]; + +const YAML_STRING = `- displayName: test-display-name + id: test-case-id + prerequisiteTestCaseId: prerequisite-test-case-id + steps: + - goal: test-goal + hint: test-hint + successCriteria: test-success-criteria +- displayName: minimal-case + id: minimal-id + steps: + - goal: win +`; + +const YAML_DATA = [ + { + displayName: "test-display-name", + id: "test-case-id", + prerequisiteTestCaseId: "prerequisite-test-case-id", + steps: [ + { + goal: "test-goal", + hint: "test-hint", + successCriteria: "test-success-criteria", + }, + ], + }, + { + displayName: "minimal-case", + id: "minimal-id", + steps: [{ goal: "win" }], + }, +]; + +describe("YamlHelper", () => { + it("converts TestCase[] to YAML string", () => { + const yamlString = toYaml(TEST_CASES); + expect(jsYaml.safeLoad(yamlString)).to.eql(YAML_DATA); + expect(yamlString).to.eq(YAML_STRING); // brittle ¯\_(ツ)_/¯ + }); + + it("converts YAML string to TestCase[]", () => { + const testCases = fromYaml(APP_NAME, YAML_STRING); + expect(testCases).to.eql(TEST_CASES); + }); + + it("converts YAML without ID", () => { + const testCases = fromYaml( + APP_NAME, + `- displayName: minimal-case + steps: + - goal: win +`, + ); + expect(testCases).to.eql([ + { + displayName: "minimal-case", + aiInstructions: { steps: [{ goal: "win" }] }, + }, + ]); + }); + + it("throws error if displayName is missing", () => { + expect(() => + fromYaml( + APP_NAME, + `- steps: + - goal: test-goal + hint: test-hint + successCriteria: test-success-criteria +`, + ), + ).to.throw(/"displayName" is required/); + }); + + it("throws error if steps is missing", () => { + expect(() => fromYaml(APP_NAME, `- displayName: test-display-name`)).to.throw( + /"steps" is required/, + ); + }); + + it("throws error if goal is missing", () => { + expect(() => + fromYaml( + APP_NAME, + `- displayName: test-display-name + steps: + - hint: test-hint + successCriteria: test-success-criteria +`, + ), + ).to.throw(/"goal" is required/); + }); + + it("throws error if additional property is present in test case", () => { + expect(() => + fromYaml( + APP_NAME, + `- displayName: test-display-name + extraTestCaseProperty: property + steps: + - goal: test-goal +`, + ), + ).to.throw(/unexpected property "extraTestCaseProperty"/); + }); + + it("throws error if additional property is present in step", () => { + expect(() => + fromYaml( + APP_NAME, + `- displayName: test-display-name + steps: + - goal: test-goal + extraStepProperty: property +`, + ), + ).to.throw(/unexpected property "extraStepProperty"/); + }); + + it("throws error if YAML is invalid", () => { + expect(() => + fromYaml( + APP_NAME, + `- +this is not valid YAML`, + ), + ).to.throw(/at line 2/); + }); + + it("throws error if YAML doesn't contain a top-level array", () => { + expect(() => fromYaml(APP_NAME, "not a list")).to.throw(/must contain a list of test cases/); + }); +}); diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts new file mode 100644 index 00000000000..466467dafdc --- /dev/null +++ b/src/appdistribution/yaml_helper.ts @@ -0,0 +1,103 @@ +import * as jsYaml from "js-yaml"; +import { getErrMsg, FirebaseError } from "../error"; +import { TestCase } from "./types"; + +declare interface YamlStep { + goal?: string; + hint?: string; + successCriteria?: string; +} + +const ALLOWED_YAML_STEP_KEYS = new Set(["goal", "hint", "successCriteria"]); + +declare interface YamlTestCase { + displayName?: string; + id?: string; + prerequisiteTestCaseId?: string; + steps?: YamlStep[]; +} + +const ALLOWED_YAML_TEST_CASE_KEYS = new Set([ + "displayName", + "id", + "prerequisiteTestCaseId", + "steps", +]); + +function extractIdFromResourceName(name: string): string { + return name.split("/").pop() ?? ""; +} + +function toYamlTestCases(testCases: TestCase[]): YamlTestCase[] { + return testCases.map((testCase) => ({ + displayName: testCase.displayName, + id: extractIdFromResourceName(testCase.name!), // resource name is retured by server + ...(testCase.prerequisiteTestCase && { + prerequisiteTestCaseId: extractIdFromResourceName(testCase.prerequisiteTestCase), + }), + steps: testCase.aiInstructions.steps.map((step) => ({ + goal: step.goal, + ...(step.hint && { hint: step.hint }), + ...(step.successCriteria && { successCriteria: step.successCriteria }), + })), + })); +} + +export function toYaml(testCases: TestCase[]): string { + return jsYaml.safeDump(toYamlTestCases(testCases)); +} + +function castExists(it: T | null | undefined, thing: string): T { + if (it == null) { + throw new FirebaseError(`"${thing}" is required`); + } + return it!; +} + +function checkAllowedKeys(allowedKeys: Set, o: object) { + for (const key of Object.keys(o)) { + if (!allowedKeys.has(key)) { + throw new FirebaseError(`unexpected property "${key}"`); + } + } +} + +function fromYamlTestCases(appName: string, yamlTestCases: YamlTestCase[]): TestCase[] { + return yamlTestCases.map((yamlTestCase) => { + checkAllowedKeys(ALLOWED_YAML_TEST_CASE_KEYS, yamlTestCase); + return { + displayName: castExists(yamlTestCase.displayName, "displayName"), + aiInstructions: { + steps: castExists(yamlTestCase.steps, "steps").map((yamlStep) => { + checkAllowedKeys(ALLOWED_YAML_STEP_KEYS, yamlStep); + return { + goal: castExists(yamlStep.goal, "goal"), + ...(yamlStep.hint && { hint: yamlStep.hint }), + ...(yamlStep.successCriteria && { + successCriteria: yamlStep.successCriteria, + }), + }; + }), + }, + ...(yamlTestCase.id && { + name: `${appName}/testCases/${yamlTestCase.id}`, + }), + ...(yamlTestCase.prerequisiteTestCaseId && { + prerequisiteTestCase: `${appName}/testCases/${yamlTestCase.prerequisiteTestCaseId}`, + }), + }; + }); +} + +export function fromYaml(appName: string, yaml: string): TestCase[] { + let parsedYaml: unknown; + try { + parsedYaml = jsYaml.safeLoad(yaml); + } catch (err: unknown) { + throw new FirebaseError(`Failed to parse YAML: ${getErrMsg(err)}`); + } + if (!Array.isArray(parsedYaml)) { + throw new FirebaseError("YAML file must contain a list of test cases."); + } + return fromYamlTestCases(appName, parsedYaml as YamlTestCase[]); +} diff --git a/src/commands/appdistribution-testcases-export.ts b/src/commands/appdistribution-testcases-export.ts new file mode 100644 index 00000000000..bfdc2aa0c6e --- /dev/null +++ b/src/commands/appdistribution-testcases-export.ts @@ -0,0 +1,30 @@ +import * as fs from "fs-extra"; +import { Command } from "../command"; +import { toYaml } from "../appdistribution/yaml_helper"; +import { requireAuth } from "../requireAuth"; +import { AppDistributionClient } from "../appdistribution/client"; +import { getAppName } from "../appdistribution/options-parser-util"; +import { TestCase, ListTestCasesResponse } from "../appdistribution/types"; +import { FirebaseError } from "../error"; +import * as utils from "../utils"; + +export const command = new Command("appdistribution:testcases:export ") + .description("export test cases as a YAML file") + .option("--app ", "the app id of your Firebase app") + .before(requireAuth) + .action(async (yamlFile: string, options?: any): Promise => { + const appName = getAppName(options); + const appDistroClient = new AppDistributionClient(); + let testCases: TestCase[]; + try { + testCases = await appDistroClient.listTestCases(appName); + } catch (err: any) { + throw new FirebaseError("Failed to list test cases.", { + exit: 1, + original: err, + }); + } + fs.writeFileSync(yamlFile, toYaml(testCases), "utf8"); + utils.logSuccess(`Exported ${testCases.length} test cases to ${yamlFile}`); + return { testCases }; + }); diff --git a/src/commands/appdistribution-testcases-import.ts b/src/commands/appdistribution-testcases-import.ts new file mode 100644 index 00000000000..343ef25ee11 --- /dev/null +++ b/src/commands/appdistribution-testcases-import.ts @@ -0,0 +1,41 @@ +import * as fs from "fs-extra"; +import { Command } from "../command"; +import { fromYaml } from "../appdistribution/yaml_helper"; +import { requireAuth } from "../requireAuth"; +import { AppDistributionClient } from "../appdistribution/client"; +import { getAppName, ensureFileExists } from "../appdistribution/options-parser-util"; +import { TestCase } from "../appdistribution/types"; +import * as utils from "../utils"; +import { FirebaseError } from "../error"; + +export const command = new Command("appdistribution:testcases:import ") + .description("import test cases from YAML file") + .option("--app ", "the app id of your Firebase app") + .before(requireAuth) + .action(async (yamlFile: string, options?: any) => { + const appName = getAppName(options); + const appDistroClient = new AppDistributionClient(); + ensureFileExists(yamlFile); + const testCases: TestCase[] = fromYaml(appName, fs.readFileSync(yamlFile, "utf8")); + const testCasesWithoutName = testCases.filter((tc) => !tc.name); + // nothing can depend on these test cases yet, since they don't have + // resource names + const creationResults = await Promise.allSettled( + testCasesWithoutName.map((tc) => appDistroClient.createTestCase(appName, tc)), + ); + const failed = creationResults.filter((r) => r.status === "rejected"); + if (failed.length > 0) { + for (const f of failed) { + utils.logWarning((f as any).reason); + } + const succeeded = creationResults.length - failed.length; + throw new FirebaseError( + `Created ${succeeded} test case(s), but failed to create ${failed.length}.`, + ); + } + // any test case with a resource name can be referenced by any other new or + // existing test case, so we need to batch the upsert. + const testCasesWithName = testCases.filter((tc) => !!tc.name); + await appDistroClient.batchUpsertTestCases(appName, testCasesWithName); + utils.logSuccess(`Imported ${testCases.length} test cases from ${yamlFile}`); + }); diff --git a/src/commands/index.ts b/src/commands/index.ts index e18f1aa23f8..1a550749e12 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -30,6 +30,9 @@ export function load(client: any): any { client.appdistribution.groups.create = loadCommand("appdistribution-groups-create"); client.appdistribution.groups.delete = loadCommand("appdistribution-groups-delete"); client.appdistribution.group = client.appdistribution.groups; + client.appdistribution.testCases = {}; + client.appdistribution.testCases.export = loadCommand("appdistribution-testcases-export"); + client.appdistribution.testCases.import = loadCommand("appdistribution-testcases-import"); client.apps = {}; client.apps.create = loadCommand("apps-create"); client.apps.list = loadCommand("apps-list");