Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for AI Testing Agent #8001

Merged
merged 3 commits into from
Dec 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/appdistribution/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,15 +110,15 @@

try {
await this.appDistroV1Client.post(`/${releaseName}:distribute`, data);
} catch (err: any) {

Check warning on line 113 in src/appdistribution/client.ts

View workflow job for this annotation

GitHub Actions / lint (20)

Unexpected any. Specify a different type
let errorMessage = err.message;

Check warning on line 114 in src/appdistribution/client.ts

View workflow job for this annotation

GitHub Actions / lint (20)

Unsafe assignment of an `any` value

Check warning on line 114 in src/appdistribution/client.ts

View workflow job for this annotation

GitHub Actions / lint (20)

Unsafe member access .message on an `any` value
const errorStatus = err?.context?.body?.error?.status;

Check warning on line 115 in src/appdistribution/client.ts

View workflow job for this annotation

GitHub Actions / lint (20)

Unsafe assignment of an `any` value

Check warning on line 115 in src/appdistribution/client.ts

View workflow job for this annotation

GitHub Actions / lint (20)

Unsafe member access .context on an `any` value
if (errorStatus === "FAILED_PRECONDITION") {
errorMessage = "invalid testers";
} else if (errorStatus === "INVALID_ARGUMENT") {
errorMessage = "invalid groups";
}
throw new FirebaseError(`failed to distribute to testers/groups: ${errorMessage}`, {

Check warning on line 121 in src/appdistribution/client.ts

View workflow job for this annotation

GitHub Actions / lint (20)

Invalid type "any" of template literal expression
exit: 1,
});
}
Expand Down Expand Up @@ -149,7 +149,7 @@
queryParams,
});
} catch (err) {
throw new FirebaseError(`Client request failed to list testers ${err}`);

Check warning on line 152 in src/appdistribution/client.ts

View workflow job for this annotation

GitHub Actions / lint (20)

Invalid type "unknown" of template literal expression
}

for (const t of apiResponse.body.testers) {
Expand Down Expand Up @@ -215,7 +215,7 @@
listGroupsResponse.groups.push(...(apiResponse.body.groups || []));
pageToken = apiResponse.body.nextPageToken;
} catch (err) {
throw new FirebaseError(`Client failed to list groups ${err}`);

Check warning on line 218 in src/appdistribution/client.ts

View workflow job for this annotation

GitHub Actions / lint (20)

Invalid type "unknown" of template literal expression
}
} while (pageToken);
return listGroupsResponse;
Expand Down Expand Up @@ -281,6 +281,7 @@
releaseName: string,
devices: TestDevice[],
loginCredential?: LoginCredential,
testCaseName?: string,
): Promise<ReleaseTest> {
try {
const response = await this.appDistroV1AlphaClient.request<ReleaseTest, ReleaseTest>({
Expand All @@ -289,6 +290,7 @@
body: {
deviceExecutions: devices.map(mapDeviceToExecution),
loginCredential,
testCase: testCaseName,
},
});
return response.body;
Expand Down
12 changes: 6 additions & 6 deletions src/appdistribution/options-parser-util.spec.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { expect } from "chai";
import { getLoginCredential, getTestDevices } from "./options-parser-util";
import { getLoginCredential, parseTestDevices } from "./options-parser-util";
import { FirebaseError } from "../error";
import * as fs from "fs-extra";
import { rmSync } from "node:fs";
Expand All @@ -21,7 +21,7 @@ describe("options-parser-util", () => {
it("parses a test device", () => {
const optionValue = "model=modelname,version=123,orientation=landscape,locale=en_US";

const result = getTestDevices(optionValue, "");
const result = parseTestDevices(optionValue, "");

expect(result).to.deep.equal([
{
Expand All @@ -37,7 +37,7 @@ describe("options-parser-util", () => {
const optionValue =
"model=modelname,version=123,orientation=landscape,locale=en_US;model=modelname2,version=456,orientation=portrait,locale=es";

const result = getTestDevices(optionValue, "");
const result = parseTestDevices(optionValue, "");

expect(result).to.deep.equal([
{
Expand All @@ -59,7 +59,7 @@ describe("options-parser-util", () => {
const optionValue =
"model=modelname,version=123,orientation=landscape,locale=en_US\nmodel=modelname2,version=456,orientation=portrait,locale=es";

const result = getTestDevices(optionValue, "");
const result = parseTestDevices(optionValue, "");

expect(result).to.deep.equal([
{
Expand All @@ -80,7 +80,7 @@ describe("options-parser-util", () => {
it("throws an error with correct format when missing a field", () => {
const optionValue = "model=modelname,version=123,locale=en_US";

expect(() => getTestDevices(optionValue, "")).to.throw(
expect(() => parseTestDevices(optionValue, "")).to.throw(
FirebaseError,
"model=<model-id>,version=<os-version-id>,locale=<locale>,orientation=<orientation>",
);
Expand All @@ -90,7 +90,7 @@ describe("options-parser-util", () => {
const optionValue =
"model=modelname,version=123,orientation=landscape,locale=en_US,notafield=blah";

expect(() => getTestDevices(optionValue, "")).to.throw(
expect(() => parseTestDevices(optionValue, "")).to.throw(
FirebaseError,
"model, version, orientation, locale",
);
Expand Down
18 changes: 9 additions & 9 deletions src/appdistribution/options-parser-util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
import { FieldHints, LoginCredential, TestDevice } from "./types";

/**
* Takes in comma separated string or a path to a comma/new line separated file
* and converts the input into an string[] of testers or groups. Value takes precedent
* over file.
* Takes in comma-separated string or a path to a comma- or newline-separated
* file and converts the input into an string[].
* Value takes precedent over file.
*/
export function getTestersOrGroups(value: string, file: string): string[] {
export function parseIntoStringArray(value: string, file: string): string[] {
// If there is no value then the file gets parsed into a string to be split
if (!value && file) {
ensureFileExists(file);
Expand All @@ -23,8 +23,8 @@
}

/**
* Takes in a string[] or a path to a comma/new line separated file of testers emails and
* returns a string[] of emails.
* Takes in a string[] or a path to a comma- or newline-separated file of
* testers emails and returns a string[] of emails.
*/
export function getEmails(emails: string[], file: string): string[] {
if (emails.length === 0) {
Expand All @@ -36,7 +36,7 @@
}

// Ensures a the file path that the user input is valid
export function ensureFileExists(file: string, message = ""): void {

Check warning on line 39 in src/appdistribution/options-parser-util.ts

View workflow job for this annotation

GitHub Actions / lint (20)

Missing JSDoc comment
if (!fs.existsSync(file)) {
throw new FirebaseError(`File ${file} does not exist: ${message}`);
}
Expand All @@ -51,7 +51,7 @@
}

// Gets project name from project number
export async function getProjectName(options: any): Promise<string> {

Check warning on line 54 in src/appdistribution/options-parser-util.ts

View workflow job for this annotation

GitHub Actions / lint (20)

Missing JSDoc comment
const projectNumber = await needProjectNumber(options);
return `projects/${projectNumber}`;
}
Expand All @@ -67,10 +67,10 @@

/**
* Takes in comma separated string or a path to a comma/new line separated file
* and converts the input into a string[] of test device strings. Value takes precedent
* over file.
* and converts the input into a string[] of test device strings.
* Value takes precedent over file.
*/
export function getTestDevices(value: string, file: string): TestDevice[] {
export function parseTestDevices(value: string, file: string): TestDevice[] {
// If there is no value then the file gets parsed into a string to be split
if (!value && file) {
ensureFileExists(file);
Expand Down
1 change: 1 addition & 0 deletions src/appdistribution/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -126,4 +126,5 @@ export interface ReleaseTest {
name?: string;
deviceExecutions: DeviceExecution[];
loginCredential?: LoginCredential;
testCase?: string;
}
128 changes: 82 additions & 46 deletions src/commands/appdistribution-distribute.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,16 @@ import {
IntegrationState,
UploadReleaseResult,
TestDevice,
ReleaseTest,
} from "../appdistribution/types";
import { FirebaseError, getErrMsg, getErrStatus } from "../error";
import { Distribution, DistributionFileType } from "../appdistribution/distribution";
import {
ensureFileExists,
getAppName,
getLoginCredential,
getTestDevices,
getTestersOrGroups,
parseTestDevices,
parseIntoStringArray,
} from "../appdistribution/options-parser-util";

const TEST_MAX_POLLING_RETRIES = 40;
Expand All @@ -35,19 +36,21 @@ function getReleaseNotes(releaseNotes: string, releaseNotesFile: string): string
}

export const command = new Command("appdistribution:distribute <release-binary-file>")
.description("upload a release binary")
.description(
"upload a release binary and optionally distribute it to testers and run automated tests",
)
.option("--app <app_id>", "the app id of your Firebase app")
.option("--release-notes <string>", "release notes to include")
.option("--release-notes-file <file>", "path to file with release notes")
.option("--testers <string>", "a comma separated list of tester emails to distribute to")
.option("--testers <string>", "a comma-separated list of tester emails to distribute to")
.option(
"--testers-file <file>",
"path to file with a comma separated list of tester emails to distribute to",
"path to file with a comma- or newline-separated list of tester emails to distribute to",
)
.option("--groups <string>", "a comma separated list of group aliases to distribute to")
.option("--groups <string>", "a comma-separated list of group aliases to distribute to")
.option(
"--groups-file <file>",
"path to file with a comma separated list of group aliases to distribute to",
"path to file with a comma- or newline-separated list of group aliases to distribute to",
)
.option(
"--test-devices <string>",
Expand Down Expand Up @@ -75,14 +78,25 @@ export const command = new Command("appdistribution:distribute <release-binary-f
"--test-non-blocking",
"run automated tests without waiting for them to complete. Visit the Firebase console for the test results.",
)
.option("--test-case-ids <string>", "a comma-separated list of test case IDs.")
.option(
"--test-case-ids-file <file>",
"path to file with a comma- or newline-separated list of test case IDs.",
)
.before(requireAuth)
.action(async (file: string, options: any) => {
const appName = getAppName(options);
const distribution = new Distribution(file);
const releaseNotes = getReleaseNotes(options.releaseNotes, options.releaseNotesFile);
const testers = getTestersOrGroups(options.testers, options.testersFile);
const groups = getTestersOrGroups(options.groups, options.groupsFile);
const testDevices = getTestDevices(options.testDevices, options.testDevicesFile);
const testers = parseIntoStringArray(options.testers, options.testersFile);
const groups = parseIntoStringArray(options.groups, options.groupsFile);
const testCases = parseIntoStringArray(options.testCaseIds, options.testCaseIdsFile);
const testDevices = parseTestDevices(options.testDevices, options.testDevicesFile);
if (testCases.length && (options.testUsernameResource || options.testPasswordResource)) {
throw new FirebaseError(
"Password and username resource names are not supported for the AI testing agent.",
);
}
const loginCredential = getLoginCredential({
username: options.testUsername,
password: options.testPassword,
Expand Down Expand Up @@ -210,56 +224,78 @@ export const command = new Command("appdistribution:distribute <release-binary-f
await requests.distribute(releaseName, testers, groups);

// Run automated tests
if (testDevices?.length) {
utils.logBullet("starting automated tests (note: this feature is in beta)");
const releaseTest = await requests.createReleaseTest(
releaseName,
testDevices,
loginCredential,
);
utils.logSuccess(`Release test created successfully`);
if (testDevices.length) {
utils.logBullet("starting automated test (note: this feature is in beta)");
const releaseTestPromises: Promise<ReleaseTest>[] = [];
if (!testCases.length) {
// fallback to basic automated test
releaseTestPromises.push(
requests.createReleaseTest(releaseName, testDevices, loginCredential),
);
} else {
for (const testCaseId of testCases) {
releaseTestPromises.push(
requests.createReleaseTest(
releaseName,
testDevices,
loginCredential,
`${appName}/testCases/${testCaseId}`,
),
);
}
}
const releaseTests = await Promise.all(releaseTestPromises);
utils.logSuccess(`${releaseTests.length} release test(s) started successfully`);
kaibolay marked this conversation as resolved.
Show resolved Hide resolved
if (!options.testNonBlocking) {
await awaitTestResults(releaseTest.name!, requests);
await awaitTestResults(releaseTests, requests);
}
}
});

async function awaitTestResults(
releaseTestName: string,
releaseTests: ReleaseTest[],
requests: AppDistributionClient,
): Promise<void> {
const releaseTestNames = new Set(releaseTests.map((rt) => rt.name!));
for (let i = 0; i < TEST_MAX_POLLING_RETRIES; i++) {
utils.logBullet("the automated tests results are pending");
utils.logBullet(`${releaseTestNames.size} automated test results are pending...`);
await delay(TEST_POLLING_INTERVAL_MILLIS);
const releaseTest = await requests.getReleaseTest(releaseTestName);
if (releaseTest.deviceExecutions.every((e) => e.state === "PASSED")) {
utils.logSuccess("automated test(s) passed!");
return;
}
for (const execution of releaseTest.deviceExecutions) {
switch (execution.state) {
case "PASSED":
case "IN_PROGRESS":
for (const releaseTestName of releaseTestNames) {
const releaseTest = await requests.getReleaseTest(releaseTestName);
if (releaseTest.deviceExecutions.every((e) => e.state === "PASSED")) {
releaseTestNames.delete(releaseTestName);
if (releaseTestNames.size === 0) {
utils.logSuccess("Automated test(s) passed!");
return;
} else {
continue;
case "FAILED":
throw new FirebaseError(
`Automated test failed for ${deviceToString(execution.device)}: ${execution.failedReason}`,
{ exit: 1 },
);
case "INCONCLUSIVE":
throw new FirebaseError(
`Automated test inconclusive for ${deviceToString(execution.device)}: ${execution.inconclusiveReason}`,
{ exit: 1 },
);
default:
throw new FirebaseError(
`Unsupported automated test state for ${deviceToString(execution.device)}: ${execution.state}`,
{ exit: 1 },
);
}
}
for (const execution of releaseTest.deviceExecutions) {
switch (execution.state) {
case "PASSED":
case "IN_PROGRESS":
continue;
case "FAILED":
throw new FirebaseError(
`Automated test failed for ${deviceToString(execution.device)}: ${execution.failedReason}`,
{ exit: 1 },
);
case "INCONCLUSIVE":
throw new FirebaseError(
`Automated test inconclusive for ${deviceToString(execution.device)}: ${execution.inconclusiveReason}`,
{ exit: 1 },
);
default:
throw new FirebaseError(
`Unsupported automated test state for ${deviceToString(execution.device)}: ${execution.state}`,
{ exit: 1 },
);
}
}
}
}
throw new FirebaseError("It took longer than expected to process your test, please try again.", {
throw new FirebaseError("It took longer than expected to run your test(s), please try again.", {
exit: 1,
});
}
Expand Down
Loading