Add export/import of app distribution test cases as YAML. (#9397)

* Add export/import of app distribution test cases as YAML.
* Add missing App Distribution commands to README

Author: kaibolay

CHANGELOG.md

@@ -2,6 +2,7 @@
 - Added `functions.list_functions` as a MCP tool (#9369)
 - Added AI Logic to `firebase init` CLI command and `firebase_init` MCP tool. (#9185)
 - Improved error messages for Firebase AI Logic provisioning during 'firebase init' (#9377)
+- Added `appdistribution:testcases:export` and `appdistribution:testcases:import` (#9397)
 - Updated to v2.16.0 of the Data Connect emulator, which includes internal improvements.
 - Data Connect now allows executing a valid query / operation even if the other operations are invalid. (This toleration provides convenience on a best-effort basis. Some errors like invalid syntax can still cause the whole request to be rejected.)
 - Fixed enum list deserialization in Data Connect generated Dart SDKs.

README.md

@@ -87,9 +87,17 @@ These commands let you deploy and interact with your Firebase services.
 
 ### App Distribution Commands
 
-| Command                        | Description            |
-| ------------------------------ | ---------------------- |
-| **appdistribution:distribute** | Upload a distribution. |
+| Command                              | Description                                                                              |
+| ------------------------------------ | ---------------------------------------------------------------------------------------- |
+| **appdistribution:distribute**       | Upload a release binary and optionally distribute it to testers and run automated tests. |
+| **appdistribution:testers:list**     | List testers in project.                                                                 |
+| **appdistribution:testers:add**      | Add testers to project (and group, if specified via flag).                               |
+| **appdistribution:testers:remove**   | Remove testers from a project (or group, if specified via flag).                         |
+| **appdistribution:groups:list**      | List groups (of testers).                                                                |
+| **appdistribution:groups:create**    | Create a group (of testers).                                                             |
+| **appdistribution:groups:delete**    | Delete a group (of testers).                                                             |
+| **appdistribution:testcases:export** | Export test cases as a YAML file.                                                        |
+| **appdistribution:testcases:import** | Import test cases from YAML file.                                                        |
 
 ### Auth Commands
 

package.json

@@ -26,6 +26,7 @@
     "mocha": "nyc --reporter=html mocha 'src/**/*.spec.{ts,js}'",
     "prepare": "npm run clean && npm run build:publish",
     "test": "npm run lint:quiet && npm run test:compile && npm run mocha",
+    "test:appdistribution": "npm run lint:quiet && npm run test:compile && nyc mocha 'src/appdistribution/*.spec.{ts,js}'",
     "test:apptesting": "npm run lint:quiet && npm run test:compile && nyc mocha 'src/apptesting/*.spec.{ts,js}'",
     "test:client-integration": "bash ./scripts/client-integration-tests/run.sh",
     "test:compile": "tsc --project tsconfig.compile.json",

src/appdistribution/client.spec.ts

@@ -7,7 +7,7 @@ import * as sinon from "sinon";
 import * as tmp from "tmp";
 
 import { AppDistributionClient } from "./client";
-import { BatchRemoveTestersResponse, Group, TestDevice } from "./types";
+import { BatchRemoveTestersResponse, Group, TestCase, TestDevice } from "./types";
 import { appDistributionOrigin } from "../api";
 import { Distribution } from "./distribution";
 import { FirebaseError } from "../error";
@@ -501,4 +501,91 @@ describe("distribution", () => {
       expect(nock.isDone()).to.be.true;
     });
   });
+
+  describe("listTestCases", () => {
+    it("should throw error if request fails", async () => {
+      nock(appDistributionOrigin())
+        .get(`/v1alpha/${appName}/testCases`)
+        .reply(400, { error: { status: "FAILED_PRECONDITION" } });
+      await expect(appDistributionClient.listTestCases(appName)).to.be.rejectedWith(
+        FirebaseError,
+        "Client failed to list test cases",
+      );
+      expect(nock.isDone()).to.be.true;
+    });
+
+    it("should resolve with array of test cases when request succeeds", async () => {
+      const testCases: TestCase[] = [
+        {
+          name: `${appName}/testCases/tc_1`,
+          displayName: "Test Case 1",
+          aiInstructions: {
+            steps: [
+              {
+                goal: "Win at all costs",
+              },
+            ],
+          },
+        },
+        {
+          name: `${appName}/testCases/tc_2`,
+          displayName: "Test Case 2",
+          aiInstructions: { steps: [] },
+        },
+      ];
+
+      nock(appDistributionOrigin()).get(`/v1alpha/${appName}/testCases`).reply(200, {
+        testCases: testCases,
+      });
+      await expect(appDistributionClient.listTestCases(appName)).to.eventually.deep.eq(testCases);
+      expect(nock.isDone()).to.be.true;
+    });
+  });
+
+  describe("createTestCase", () => {
+    const mockTestCase = { displayName: "Case", aiInstructions: { steps: [] } };
+
+    it("should throw error if request fails", async () => {
+      nock(appDistributionOrigin())
+        .post(`/v1alpha/${appName}/testCases`)
+        .reply(400, { error: { status: "FAILED_PRECONDITION" } });
+      await expect(appDistributionClient.createTestCase(appName, mockTestCase)).to.be.rejectedWith(
+        FirebaseError,
+        "Failed to create test case",
+      );
+      expect(nock.isDone()).to.be.true;
+    });
+
+    it("should resolve with TestCase when request succeeds", async () => {
+      nock(appDistributionOrigin()).post(`/v1alpha/${appName}/testCases`).reply(200, mockTestCase);
+      await expect(
+        appDistributionClient.createTestCase(appName, mockTestCase),
+      ).to.be.eventually.deep.eq(mockTestCase);
+      expect(nock.isDone()).to.be.true;
+    });
+  });
+
+  describe("batchUpsertTestCases", () => {
+    const mockTestCase = { displayName: "Case", aiInstructions: { steps: [] } };
+
+    it("should throw error if request fails", async () => {
+      nock(appDistributionOrigin())
+        .post(`/v1alpha/${appName}/testCases:batchUpdate`)
+        .reply(400, { error: { status: "FAILED_PRECONDITION" } });
+      await expect(
+        appDistributionClient.batchUpsertTestCases(appName, [mockTestCase]),
+      ).to.be.rejectedWith(FirebaseError, "Failed to upsert test cases");
+      expect(nock.isDone()).to.be.true;
+    });
+
+    it("should resolve with TestCase when request succeeds", async () => {
+      nock(appDistributionOrigin())
+        .post(`/v1alpha/${appName}/testCases:batchUpdate`)
+        .reply(200, { testCases: [mockTestCase] });
+      await expect(
+        appDistributionClient.batchUpsertTestCases(appName, [mockTestCase]),
+      ).to.be.eventually.deep.eq([mockTestCase]);
+      expect(nock.isDone()).to.be.true;
+    });
+  });
 });

src/appdistribution/client.ts

@@ -10,12 +10,15 @@ import { appDistributionOrigin } from "../api";
 import {
   AabInfo,
   BatchRemoveTestersResponse,
+  BatchUpdateTestCasesRequest,
+  BatchUpdateTestCasesResponse,
   Group,
   ListGroupsResponse,
+  ListTestCasesResponse,
   ListTestersResponse,
   LoginCredential,
-  mapDeviceToExecution,
   ReleaseTest,
+  TestCase,
   TestDevice,
   Tester,
   UploadReleaseResponse,
@@ -112,7 +115,7 @@ export class AppDistributionClient {
     try {
       await this.appDistroV1Client.post(`/${releaseName}:distribute`, data);
     } catch (err: any) {
-      let errorMessage = err.message;
+      let errorMessage = getErrMsg(err);
       const errorStatus = err?.context?.body?.error?.status;
       if (errorStatus === "FAILED_PRECONDITION") {
         errorMessage = "invalid testers";
@@ -144,8 +147,8 @@ export class AppDistributionClient {
         apiResponse = await client.get<ListTestersResponse>(`${projectName}/testers`, {
           queryParams,
         });
-      } catch (err) {
-        throw new FirebaseError(`Client request failed to list testers ${err}`);
+      } catch (err: unknown) {
+        throw new FirebaseError(`Client request failed to list testers ${getErrMsg(err)}`);
       }
 
       for (const t of apiResponse.body.testers ?? []) {
@@ -206,8 +209,8 @@ export class AppDistributionClient {
         });
         groups.push(...(apiResponse.body.groups ?? []));
         pageToken = apiResponse.body.nextPageToken;
-      } catch (err) {
-        throw new FirebaseError(`Client failed to list groups ${err}`);
+      } catch (err: unknown) {
+        throw new FirebaseError(`Client failed to list groups ${getErrMsg(err)}`);
       }
     } while (pageToken);
     return groups;
@@ -280,7 +283,7 @@ export class AppDistributionClient {
         method: "POST",
         path: `${releaseName}/tests`,
         body: {
-          deviceExecutions: devices.map(mapDeviceToExecution),
+          deviceExecutions: devices.map((device) => ({ device })),
           loginCredential,
           testCase: testCaseName,
         },
@@ -295,4 +298,58 @@ export class AppDistributionClient {
     const response = await this.appDistroV1AlphaClient.get<ReleaseTest>(releaseTestName);
     return response.body;
   }
+
+  async listTestCases(appName: string): Promise<TestCase[]> {
+    const testCases: TestCase[] = [];
+    const client = this.appDistroV1AlphaClient;
+
+    let pageToken: string | undefined;
+    do {
+      const queryParams: Record<string, string> = pageToken ? { pageToken } : {};
+      try {
+        const apiResponse = await client.get<ListTestCasesResponse>(`${appName}/testCases`, {
+          queryParams,
+        });
+        testCases.push(...(apiResponse.body.testCases ?? []));
+        pageToken = apiResponse.body.nextPageToken;
+      } catch (err: unknown) {
+        throw new FirebaseError(`Client failed to list test cases ${getErrMsg(err)}`);
+      }
+    } while (pageToken);
+    return testCases;
+  }
+
+  async createTestCase(appName: string, testCase: TestCase): Promise<TestCase> {
+    try {
+      const response = await this.appDistroV1AlphaClient.request<TestCase, TestCase>({
+        method: "POST",
+        path: `${appName}/testCases`,
+        body: testCase,
+      });
+      return response.body;
+    } catch (err: unknown) {
+      throw new FirebaseError(`Failed to create test case ${getErrMsg(err)}`);
+    }
+  }
+
+  async batchUpsertTestCases(appName: string, testCases: TestCase[]): Promise<TestCase[]> {
+    try {
+      const response = await this.appDistroV1AlphaClient.request<
+        BatchUpdateTestCasesRequest,
+        BatchUpdateTestCasesResponse
+      >({
+        method: "POST",
+        path: `${appName}/testCases:batchUpdate`,
+        body: {
+          requests: testCases.map((tc) => ({
+            testCase: tc,
+            allowMissing: true,
+          })),
+        },
+      });
+      return response.body.testCases;
+    } catch (err: unknown) {
+      throw new FirebaseError(`Failed to upsert test cases ${getErrMsg(err)}`);
+    }
+  }
 }

src/appdistribution/types.ts

@@ -100,17 +100,6 @@ export interface DeviceExecution {
   inconclusiveReason?: string;
 }
 
-export function mapDeviceToExecution(device: TestDevice): DeviceExecution {
-  return {
-    device: {
-      model: device.model,
-      version: device.version,
-      orientation: device.orientation,
-      locale: device.locale,
-    },
-  };
-}
-
 export interface FieldHints {
   usernameResourceName?: string;
   passwordResourceName?: string;
@@ -128,3 +117,38 @@ export interface ReleaseTest {
   loginCredential?: LoginCredential;
   testCase?: string;
 }
+
+export interface AiStep {
+  goal: string;
+  hint?: string;
+  successCriteria?: string;
+}
+
+export interface AiInstructions {
+  steps: AiStep[];
+}
+
+export interface TestCase {
+  name?: string;
+  displayName: string;
+  prerequisiteTestCase?: string;
+  aiInstructions: AiInstructions;
+}
+
+export interface ListTestCasesResponse {
+  testCases: TestCase[];
+  nextPageToken?: string;
+}
+
+export interface UpdateTestCaseRequest {
+  testCase: TestCase;
+  allowMissing?: boolean;
+}
+
+export interface BatchUpdateTestCasesRequest {
+  requests: UpdateTestCaseRequest[];
+}
+
+export interface BatchUpdateTestCasesResponse {
+  testCases: TestCase[];
+}