diff --git a/client/components/ManageBotRunDialog/queries.js b/client/components/ManageBotRunDialog/queries.js index dba46068e..674c6856b 100644 --- a/client/components/ManageBotRunDialog/queries.js +++ b/client/components/ManageBotRunDialog/queries.js @@ -52,6 +52,9 @@ export const RETRY_CANCELED_COLLECTIONS = gql` retryCanceledCollections { id status + testStatus { + status + } } } } diff --git a/server/models/services/CollectionJobService.js b/server/models/services/CollectionJobService.js index 8d640dc40..774a51e52 100644 --- a/server/models/services/CollectionJobService.js +++ b/server/models/services/CollectionJobService.js @@ -388,12 +388,13 @@ const getCollectionJobs = async ({ const triggerWorkflow = async (job, testIds, atVersion, { transaction }) => { const { testPlanVersion } = job.testPlanRun.testPlanReport; const { gitSha, directory } = testPlanVersion; + try { if (isGithubWorkflowEnabled()) { // TODO: pass the reduced list of testIds along / deal with them somehow await createGithubWorkflow({ job, directory, gitSha, atVersion }); } else { - await startCollectionJobSimulation(job, atVersion, transaction); + await startCollectionJobSimulation(job, testIds, atVersion, transaction); } } catch (error) { console.error(error); @@ -479,14 +480,12 @@ const retryCanceledCollections = async ({ collectionJob }, { transaction }) => { throw new Error('collectionJob is required to retry cancelled tests'); } - const cancelledTests = collectionJob.testPlanRun.testResults.filter( - testResult => - // Find tests that don't have complete output - !testResult?.scenarioResults?.every(scenario => scenario?.output !== null) + const cancelledTests = collectionJob.testStatus.filter( + testStatus => testStatus.status === COLLECTION_JOB_STATUS.CANCELLED ); const testPlanReport = await getTestPlanReportById({ - id: job.testPlanRun.testPlanReportId, + id: collectionJob.testPlanRun.testPlanReportId, transaction }); @@ -497,10 +496,22 @@ const retryCanceledCollections = async ({ collectionJob }, { transaction }) => { transaction ); - const testIds = cancelledTests.map(test => test.id); + const testIds = cancelledTests.map(test => test.testId); - const job = await getCollectionJobById({ + const job = await updateCollectionJobById({ id: collectionJob.id, + values: { status: COLLECTION_JOB_STATUS.QUEUED }, + transaction + }); + + await updateCollectionJobTestStatusByQuery({ + where: { + collectionJobId: job.id, + status: COLLECTION_JOB_STATUS.CANCELLED + }, + values: { + status: COLLECTION_JOB_STATUS.QUEUED + }, transaction }); @@ -652,6 +663,15 @@ const restartCollectionJob = async ({ id }, { transaction }) => { }, transaction }); + await updateCollectionJobTestStatusByQuery({ + where: { + collectionJobId: id + }, + values: { + status: COLLECTION_JOB_STATUS.QUEUED + }, + transaction + }); if (!job) { return null; @@ -669,7 +689,16 @@ const restartCollectionJob = async ({ id }, { transaction }) => { transaction ); - return triggerWorkflow(job, [], atVersion, { transaction }); + const tests = await runnableTestsResolver(testPlanReport, null, { + transaction + }); + + return triggerWorkflow( + job, + tests.map(test => test.id), + atVersion, + { transaction } + ); }; /** diff --git a/server/tests/integration/automation-scheduler.test.js b/server/tests/integration/automation-scheduler.test.js index 4ec8e4be3..60a53b191 100644 --- a/server/tests/integration/automation-scheduler.test.js +++ b/server/tests/integration/automation-scheduler.test.js @@ -1,7 +1,8 @@ const startSupertestServer = require('../util/api-server'); const automationRoutes = require('../../routes/automation'); const { - setupMockAutomationSchedulerServer + setupMockAutomationSchedulerServer, + startCollectionJobSimulation } = require('../util/mock-automation-scheduler-server'); const db = require('../../models/index'); const { query, mutate } = require('../util/graphql-test-utilities'); @@ -197,6 +198,22 @@ const restartCollectionJobByMutation = async (jobId, { transaction }) => { transaction } ); +const retryCanceledCollectionJobByMutation = async (jobId, { transaction }) => + await mutate( + ` + mutation { + collectionJob(id: "${jobId}") { + retryCanceledCollections { + id + status + testStatus { + status + } + } + } + } `, + { transaction } + ); const cancelCollectionJobByMutation = async (jobId, { transaction }) => await mutate( ` @@ -237,6 +254,18 @@ describe('Automation controller', () => { expect(storedJob.status).toEqual('QUEUED'); expect(storedJob.testPlanRun.testPlanReport.id).toEqual(testPlanReportId); expect(storedJob.testPlanRun.testResults.length).toEqual(0); + const collectionJob = await getCollectionJobById({ + id: storedJob.id, + transaction + }); + const tests = + collectionJob.testPlanRun.testPlanReport.testPlanVersion.tests.filter( + test => test.at.key === 'voiceover_macos' + ); + // check testIds - order doesn't matter, so we sort them + expect( + startCollectionJobSimulation.lastCallParams.testIds.sort() + ).toEqual(tests.map(test => test.id).sort()); }); }); @@ -260,6 +289,105 @@ describe('Automation controller', () => { }); }); + it('should retry a cancelled job with only remaining tests', async () => { + await apiServer.sessionAgentDbCleaner(async transaction => { + const { scheduleCollectionJob: job } = + await scheduleCollectionJobByMutation({ transaction }); + const collectionJob = await getCollectionJobById({ + id: job.id, + transaction + }); + const secret = await getJobSecret(collectionJob.id, { transaction }); + + // start "RUNNING" the job + const response1 = await sessionAgent + .post(`/api/jobs/${collectionJob.id}`) + .send({ status: 'RUNNING' }) + .set('x-automation-secret', secret) + .set('x-transaction-id', transaction.id); + expect(response1.statusCode).toBe(200); + + // simulate a response for a test + const automatedTestResponse = 'AUTOMATED TEST RESPONSE'; + const ats = await AtLoader().getAll({ transaction }); + const browsers = await BrowserLoader().getAll({ transaction }); + const at = ats.find( + at => at.id === collectionJob.testPlanRun.testPlanReport.at.id + ); + const browser = browsers.find( + browser => + browser.id === collectionJob.testPlanRun.testPlanReport.browser.id + ); + const { tests } = + collectionJob.testPlanRun.testPlanReport.testPlanVersion; + const selectedTestIndex = 2; + + const selectedTest = tests[selectedTestIndex]; + const selectedTestRowNumber = selectedTest.rowNumber; + + const numberOfScenarios = selectedTest.scenarios.filter( + scenario => scenario.atId === at.id + ).length; + const response2 = await sessionAgent + .post(`/api/jobs/${collectionJob.id}/test/${selectedTestRowNumber}`) + .send({ + capabilities: { + atName: at.name, + atVersion: at.atVersions[0].name, + browserName: browser.name, + browserVersion: browser.browserVersions[0].name + }, + responses: new Array(numberOfScenarios).fill(automatedTestResponse) + }) + .set('x-automation-secret', secret) + .set('x-transaction-id', transaction.id); + expect(response2.statusCode).toBe(200); + // cancel the job + const { + collectionJob: { cancelCollectionJob: cancelledCollectionJob } + } = await cancelCollectionJobByMutation(collectionJob.id, { + transaction + }); + + // check canceled status + + expect(cancelledCollectionJob.status).toEqual('CANCELLED'); + const { collectionJob: storedCollectionJob } = await getTestCollectionJob( + collectionJob.id, + { transaction } + ); + expect(storedCollectionJob.status).toEqual('CANCELLED'); + for (const test of storedCollectionJob.testStatus) { + const expectedStatus = + test.test.id == selectedTest.id ? 'COMPLETED' : 'CANCELLED'; + expect(test.status).toEqual(expectedStatus); + } + + // retry job + const data = await retryCanceledCollectionJobByMutation( + collectionJob.id, + { transaction } + ); + expect(data.collectionJob.retryCanceledCollections.status).toBe('QUEUED'); + const { collectionJob: restartedCollectionJob } = + await getTestCollectionJob(collectionJob.id, { transaction }); + // check restarted status + expect(restartedCollectionJob.status).toEqual('QUEUED'); + for (const test of restartedCollectionJob.testStatus) { + const expectedStatus = + test.test.id == selectedTest.id ? 'COMPLETED' : 'QUEUED'; + expect(test.status).toEqual(expectedStatus); + } + const expectedTests = tests.filter( + test => test.at.key === 'voiceover_macos' && test.id != selectedTest.id + ); + // check testIds - order doesn't matter, so we sort them + expect( + startCollectionJobSimulation.lastCallParams.testIds.sort() + ).toEqual(expectedTests.map(test => test.id).sort()); + }); + }); + it('should gracefully reject request to cancel a job that does not exist', async () => { await dbCleaner(async transaction => { expect.assertions(1); // Make sure an assertion is made diff --git a/server/tests/util/mock-automation-scheduler-server.js b/server/tests/util/mock-automation-scheduler-server.js index b92b6890d..232fa7b4a 100644 --- a/server/tests/util/mock-automation-scheduler-server.js +++ b/server/tests/util/mock-automation-scheduler-server.js @@ -136,10 +136,18 @@ const simulateResultCompletion = async ( } }; -const startCollectionJobSimulation = async (job, atVersion, transaction) => { +const startCollectionJobSimulation = async ( + job, + testIds, + atVersion, + transaction +) => { if (!mockSchedulerEnabled) throw new Error('mock scheduler is not enabled'); if (process.env.ENVIRONMENT === 'test') { // stub behavior in test suite + + startCollectionJobSimulation.lastCallParams = { job, testIds, atVersion }; + return { status: COLLECTION_JOB_STATUS.QUEUED }; } else { const { data } = await apolloServer.executeOperation(