From fb97a869d572a04386046c06f286499aa1dc616e Mon Sep 17 00:00:00 2001 From: Kai Bolay Date: Tue, 26 Nov 2024 11:44:05 -0500 Subject: [PATCH] Add support for `--test-case-ids` and `--test-case-ids-file` to `appdistribution:distribute` to launch AI Testing Agent. --- src/appdistribution/client.ts | 2 + .../options-parser-util.spec.ts | 12 +- src/appdistribution/options-parser-util.ts | 14 +- src/appdistribution/types.ts | 1 + src/commands/appdistribution-distribute.ts | 128 +++++++++++------- 5 files changed, 98 insertions(+), 59 deletions(-) diff --git a/src/appdistribution/client.ts b/src/appdistribution/client.ts index 512b9bccf23a..c11cfdd2f890 100644 --- a/src/appdistribution/client.ts +++ b/src/appdistribution/client.ts @@ -281,6 +281,7 @@ export class AppDistributionClient { releaseName: string, devices: TestDevice[], loginCredential?: LoginCredential, + testCaseName?: string, ): Promise { try { const response = await this.appDistroV1AlphaClient.request({ @@ -289,6 +290,7 @@ export class AppDistributionClient { body: { deviceExecutions: devices.map(mapDeviceToExecution), loginCredential, + testCase: testCaseName, }, }); return response.body; diff --git a/src/appdistribution/options-parser-util.spec.ts b/src/appdistribution/options-parser-util.spec.ts index 41892902d5f7..0f8771a7b408 100644 --- a/src/appdistribution/options-parser-util.spec.ts +++ b/src/appdistribution/options-parser-util.spec.ts @@ -1,5 +1,5 @@ import { expect } from "chai"; -import { getLoginCredential, getTestDevices } from "./options-parser-util"; +import { getLoginCredential, parseTestDevicesFromStringOrFile } from "./options-parser-util"; import { FirebaseError } from "../error"; import * as fs from "fs-extra"; import { rmSync } from "node:fs"; @@ -21,7 +21,7 @@ describe("options-parser-util", () => { it("parses a test device", () => { const optionValue = "model=modelname,version=123,orientation=landscape,locale=en_US"; - const result = getTestDevices(optionValue, ""); + const result = parseTestDevicesFromStringOrFile(optionValue, ""); expect(result).to.deep.equal([ { @@ -37,7 +37,7 @@ describe("options-parser-util", () => { const optionValue = "model=modelname,version=123,orientation=landscape,locale=en_US;model=modelname2,version=456,orientation=portrait,locale=es"; - const result = getTestDevices(optionValue, ""); + const result = parseTestDevicesFromStringOrFile(optionValue, ""); expect(result).to.deep.equal([ { @@ -59,7 +59,7 @@ describe("options-parser-util", () => { const optionValue = "model=modelname,version=123,orientation=landscape,locale=en_US\nmodel=modelname2,version=456,orientation=portrait,locale=es"; - const result = getTestDevices(optionValue, ""); + const result = parseTestDevicesFromStringOrFile(optionValue, ""); expect(result).to.deep.equal([ { @@ -80,7 +80,7 @@ describe("options-parser-util", () => { it("throws an error with correct format when missing a field", () => { const optionValue = "model=modelname,version=123,locale=en_US"; - expect(() => getTestDevices(optionValue, "")).to.throw( + expect(() => parseTestDevicesFromStringOrFile(optionValue, "")).to.throw( FirebaseError, "model=,version=,locale=,orientation=", ); @@ -90,7 +90,7 @@ describe("options-parser-util", () => { const optionValue = "model=modelname,version=123,orientation=landscape,locale=en_US,notafield=blah"; - expect(() => getTestDevices(optionValue, "")).to.throw( + expect(() => parseTestDevicesFromStringOrFile(optionValue, "")).to.throw( FirebaseError, "model, version, orientation, locale", ); diff --git a/src/appdistribution/options-parser-util.ts b/src/appdistribution/options-parser-util.ts index dd42a68aba95..d9fdee63e9a8 100644 --- a/src/appdistribution/options-parser-util.ts +++ b/src/appdistribution/options-parser-util.ts @@ -4,11 +4,11 @@ import { needProjectNumber } from "../projectUtils"; import { FieldHints, LoginCredential, TestDevice } from "./types"; /** - * Takes in comma separated string or a path to a comma/new line separated file - * and converts the input into an string[] of testers or groups. Value takes precedent - * over file. + * Takes in comma-separated string or a path to a comma- or newline-separated + * file and converts the input into an string[]. + * Value takes precedent over file. */ -export function getTestersOrGroups(value: string, file: string): string[] { +export function parseIntoStringArray(value: string, file: string): string[] { // If there is no value then the file gets parsed into a string to be split if (!value && file) { ensureFileExists(file); @@ -23,8 +23,8 @@ export function getTestersOrGroups(value: string, file: string): string[] { } /** - * Takes in a string[] or a path to a comma/new line separated file of testers emails and - * returns a string[] of emails. + * Takes in a string[] or a path to a comma- or newline-separated file of + * testers emails and returns a string[] of emails. */ export function getEmails(emails: string[], file: string): string[] { if (emails.length === 0) { @@ -70,7 +70,7 @@ export function getAppName(options: any): string { * and converts the input into a string[] of test device strings. Value takes precedent * over file. */ -export function getTestDevices(value: string, file: string): TestDevice[] { +export function parseTestDevicesFromStringOrFile(value: string, file: string): TestDevice[] { // If there is no value then the file gets parsed into a string to be split if (!value && file) { ensureFileExists(file); diff --git a/src/appdistribution/types.ts b/src/appdistribution/types.ts index 4ba66b8e5af1..6c3db213accf 100644 --- a/src/appdistribution/types.ts +++ b/src/appdistribution/types.ts @@ -126,4 +126,5 @@ export interface ReleaseTest { name?: string; deviceExecutions: DeviceExecution[]; loginCredential?: LoginCredential; + testCase?: string; } diff --git a/src/commands/appdistribution-distribute.ts b/src/commands/appdistribution-distribute.ts index 53f490e8cee7..0160f233e2d6 100644 --- a/src/commands/appdistribution-distribute.ts +++ b/src/commands/appdistribution-distribute.ts @@ -9,6 +9,7 @@ import { IntegrationState, UploadReleaseResult, TestDevice, + ReleaseTest, } from "../appdistribution/types"; import { FirebaseError, getErrMsg, getErrStatus } from "../error"; import { Distribution, DistributionFileType } from "../appdistribution/distribution"; @@ -16,8 +17,8 @@ import { ensureFileExists, getAppName, getLoginCredential, - getTestDevices, - getTestersOrGroups, + parseTestDevicesFromStringOrFile as parseTestDevices, + parseIntoStringArray as parseIntoStringArray, } from "../appdistribution/options-parser-util"; const TEST_MAX_POLLING_RETRIES = 40; @@ -35,19 +36,21 @@ function getReleaseNotes(releaseNotes: string, releaseNotesFile: string): string } export const command = new Command("appdistribution:distribute ") - .description("upload a release binary") + .description( + "upload a release binary, optionally distribute it to testers and/or run automated tests", + ) .option("--app ", "the app id of your Firebase app") .option("--release-notes ", "release notes to include") .option("--release-notes-file ", "path to file with release notes") - .option("--testers ", "a comma separated list of tester emails to distribute to") + .option("--testers ", "a comma-separated list of tester emails to distribute to") .option( "--testers-file ", - "path to file with a comma separated list of tester emails to distribute to", + "path to file with a comma- or newline-separated list of tester emails to distribute to", ) - .option("--groups ", "a comma separated list of group aliases to distribute to") + .option("--groups ", "a comma-separated list of group aliases to distribute to") .option( "--groups-file ", - "path to file with a comma separated list of group aliases to distribute to", + "path to file with a comma- or newline-separated list of group aliases to distribute to", ) .option( "--test-devices ", @@ -75,14 +78,25 @@ export const command = new Command("appdistribution:distribute ", "a comma-separated list of test case IDs.") + .option( + "--test-case-ids-file ", + "path to file with a comma- or newline-separated list of test case IDs.", + ) .before(requireAuth) .action(async (file: string, options: any) => { const appName = getAppName(options); const distribution = new Distribution(file); const releaseNotes = getReleaseNotes(options.releaseNotes, options.releaseNotesFile); - const testers = getTestersOrGroups(options.testers, options.testersFile); - const groups = getTestersOrGroups(options.groups, options.groupsFile); - const testDevices = getTestDevices(options.testDevices, options.testDevicesFile); + const testers = parseIntoStringArray(options.testers, options.testersFile); + const groups = parseIntoStringArray(options.groups, options.groupsFile); + const testCases = parseIntoStringArray(options.testCaseIds, options.testCaseIdsFile); + const testDevices = parseTestDevices(options.testDevices, options.testDevicesFile); + if (testCases.length && (options.testUsernameResource || options.testPasswordResource)) { + throw new FirebaseError( + "Username and password resource names are not supported for the AI testing agent.", + ); + } const loginCredential = getLoginCredential({ username: options.testUsername, password: options.testPassword, @@ -210,56 +224,78 @@ export const command = new Command("appdistribution:distribute [] = []; + if (!testCases.length) { + // fallback to basic automated test + releaseTestPromises.push( + requests.createReleaseTest(releaseName, testDevices, loginCredential), + ); + } else { + for (const testCaseId of testCases) { + releaseTestPromises.push( + requests.createReleaseTest( + releaseName, + testDevices, + loginCredential, + `${appName}/testCases/${testCaseId}`, + ), + ); + } + } + const releaseTests = await Promise.all(releaseTestPromises); + utils.logSuccess(`${releaseTests.length} release test(s) started successfully`); if (!options.testNonBlocking) { - await awaitTestResults(releaseTest.name!, requests); + const releaseTestNames = new Set(releaseTests.map((rt) => rt.name!!)); + await awaitTestResults(releaseTestNames, requests); } } }); async function awaitTestResults( - releaseTestName: string, + releaseTestNames: Set, requests: AppDistributionClient, ): Promise { for (let i = 0; i < TEST_MAX_POLLING_RETRIES; i++) { - utils.logBullet("the automated tests results are pending"); + utils.logBullet("the automated test results are pending"); await delay(TEST_POLLING_INTERVAL_MILLIS); - const releaseTest = await requests.getReleaseTest(releaseTestName); - if (releaseTest.deviceExecutions.every((e) => e.state === "PASSED")) { - utils.logSuccess("automated test(s) passed!"); - return; - } - for (const execution of releaseTest.deviceExecutions) { - switch (execution.state) { - case "PASSED": - case "IN_PROGRESS": + for (const releaseTestName of releaseTestNames) { + const releaseTest = await requests.getReleaseTest(releaseTestName); + if (releaseTest.deviceExecutions.every((e) => e.state === "PASSED")) { + releaseTestNames.delete(releaseTestName); + if (releaseTestNames.size === 0) { + utils.logSuccess("automated test(s) passed!"); + return; + } else { continue; - case "FAILED": - throw new FirebaseError( - `Automated test failed for ${deviceToString(execution.device)}: ${execution.failedReason}`, - { exit: 1 }, - ); - case "INCONCLUSIVE": - throw new FirebaseError( - `Automated test inconclusive for ${deviceToString(execution.device)}: ${execution.inconclusiveReason}`, - { exit: 1 }, - ); - default: - throw new FirebaseError( - `Unsupported automated test state for ${deviceToString(execution.device)}: ${execution.state}`, - { exit: 1 }, - ); + } + } + for (const execution of releaseTest.deviceExecutions) { + switch (execution.state) { + case "PASSED": + case "IN_PROGRESS": + continue; + case "FAILED": + throw new FirebaseError( + `Automated test failed for ${deviceToString(execution.device)}: ${execution.failedReason}`, + { exit: 1 }, + ); + case "INCONCLUSIVE": + throw new FirebaseError( + `Automated test inconclusive for ${deviceToString(execution.device)}: ${execution.inconclusiveReason}`, + { exit: 1 }, + ); + default: + throw new FirebaseError( + `Unsupported automated test state for ${deviceToString(execution.device)}: ${execution.state}`, + { exit: 1 }, + ); + } } } } - throw new FirebaseError("It took longer than expected to process your test, please try again.", { + throw new FirebaseError("It took longer than expected to run your test(s), please try again.", { exit: 1, }); }