Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update deduplicate migration #739

Merged
merged 5 commits into from
Aug 14, 2023
Merged

Update deduplicate migration #739

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7fa662d
Update deduplicate migration to account for test plan versions which …
howard-e Aug 7, 2023
ad12e45
Update comments; fix errors in components (already addressed in other…
howard-e Aug 7, 2023
4c28e8a
Update if block
howard-e Aug 7, 2023
ade9c40
Minor change
howard-e Aug 14, 2023
79444ce
TestPlanVersion.tests[].viewers is also variable and should be ignore…
howard-e Aug 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion client/components/TestManagement/StatusSummaryRow/index.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ const StatusSummaryRow = ({ reportResult, testPlanVersion }) => {
<LoadingStatus message={loadingMessage}>
<tr>
<th>
{testPlanVersion.title}
{testPlanVersion?.title}
{Object.entries(reportResult).length > 0 && (
<PhaseText className={phase.toLowerCase()}>
{phase}
Expand Down
2 changes: 1 addition & 1 deletion client/components/TestManagement/index.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ const TestManagement = () => {
key={
tabularReport
.latestTestPlanVersion
.id
?.id
}
testPlanVersion={
tabularReport.latestTestPlanVersion
Expand Down
2 changes: 1 addition & 1 deletion client/components/TestQueueRow/index.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ const TestQueueRow = ({
);

const latestTestPlanVersion = latestTestPlanVersions.filter(
version => version.latestTestPlanVersion.id === testPlanVersion.id
version => version.latestTestPlanVersion?.id === testPlanVersion.id
);
const updateTestPlanVersionButton = isAdmin &&
latestTestPlanVersion.length === 0 && (
Expand Down
298 changes: 286 additions & 12 deletions server/migrations/20230501220810-deduplicateTestPlanVersions.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
'use strict';

const fetch = require('node-fetch');
const {
createTestResultId,
createScenarioResultId,
Expand All @@ -15,7 +16,7 @@ module.exports = {
* hash
* @param transaction - The Sequelize.Transaction object.
* See {@https://sequelize.org/api/v6/class/src/sequelize.js~sequelize#instance-method-transaction}
* @returns {Promise<{testPlanVersionIdsByHashedTests: {}}>}
* @returns {Promise<{testPlanVersionsByHashedTests: {}}>}
*/
const computeTestPlanVersionHashedTests = async transaction => {
const results = await queryInterface.sequelize.query(
Expand All @@ -29,7 +30,7 @@ module.exports = {
testPlanVersionCount / testPlanVersionBatchSize
);

let testPlanVersionIdsByHashedTests = {};
let testPlanVersionsByHashedTests = {};

for (let i = 0; i < iterationsNeeded; i += 1) {
const multipleOf100 = i % testPlanVersionBatchSize === 0;
Expand All @@ -44,7 +45,7 @@ module.exports = {
const currentOffset = i * testPlanVersionBatchSize;

const [testPlanVersions] = await queryInterface.sequelize.query(
`SELECT id, tests FROM "TestPlanVersion" ORDER BY id LIMIT ? OFFSET ?`,
`SELECT id, directory, "gitSha", tests, "updatedAt" FROM "TestPlanVersion" ORDER BY id LIMIT ? OFFSET ?`,
{
replacements: [testPlanVersionBatchSize, currentOffset],
transaction
Expand All @@ -55,12 +56,15 @@ module.exports = {
testPlanVersions.map(async testPlanVersion => {
const hashedTests = hashTests(testPlanVersion.tests);

if (!testPlanVersionIdsByHashedTests[hashedTests]) {
testPlanVersionIdsByHashedTests[hashedTests] = [];
if (!testPlanVersionsByHashedTests[hashedTests]) {
testPlanVersionsByHashedTests[hashedTests] = [];
}
testPlanVersionIdsByHashedTests[hashedTests].push(
testPlanVersion.id
);
testPlanVersionsByHashedTests[hashedTests].push({
id: testPlanVersion.id,
gitSha: testPlanVersion.gitSha,
directory: testPlanVersion.directory,
updatedAt: testPlanVersion.updatedAt
});

await queryInterface.sequelize.query(
`UPDATE "TestPlanVersion" SET "hashedTests" = ? WHERE id = ?`,
Expand All @@ -83,7 +87,7 @@ module.exports = {
);
}

return { testPlanVersionIdsByHashedTests };
return { testPlanVersionsByHashedTests };
};

/**
Expand Down Expand Up @@ -278,6 +282,215 @@ module.exports = {
);
};

/**
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Beast mode starts here lol

* Returns an object using test directories as the key, which holds an array containing data
* on the known commits.
* Example:
* {
* alert: [ { sha: 'string', commitDate: 'dateString' }, { sha: ..., commitDate: ... }, ... ],
* banner: [ { sha: 'string', commitDate: 'dateString' }, ... ],
* ...
* }
* @returns {Promise<{}>}
*/
const getKnownGitCommits = async () => {
const testDirectories = [
'alert',
'banner',
'breadcrumb',
'checkbox',
'checkbox-tri-state',
'combobox-autocomplete-both-updated',
'combobox-select-only',
'command-button',
'complementary',
'contentinfo',
'datepicker-spin-button',
'disclosure-faq',
'disclosure-navigation',
'form',
'horizontal-slider',
'link-css',
'link-img-alt',
'link-span-text',
'main',
'menu-button-actions',
'menu-button-actions-active-descendant',
'menu-button-navigation',
'menubar-editor',
'meter',
'minimal-data-grid',
'modal-dialog',
'radiogroup-aria-activedescendant',
'radiogroup-roving-tabindex',
'rating-slider',
'seek-slider',
'slider-multithumb',
'switch',
'tabs-manual-activation',
'toggle-button',
'vertical-temperature-slider'
];
const knownGitCommits = {};

const { GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET } = process.env;

for (const testDirectory of testDirectories) {
if (!knownGitCommits[testDirectory])
knownGitCommits[testDirectory] = [];

try {
const url = `https://api.github.com/repos/w3c/aria-at/commits?path=tests/${testDirectory}`;
const authorizationHeader = `Basic ${Buffer.from(
`${GITHUB_CLIENT_ID}:${GITHUB_CLIENT_SECRET}`
).toString('base64')}`;
const options = {
headers: {
Authorization: authorizationHeader
}
};

const response = await fetch(url, options);
const data = await response.json();

for (const commitData of data) {
if (commitData.commit?.author) {
knownGitCommits[testDirectory].push({
sha: commitData.sha,
commitDate: commitData.commit.author.date
});
}
}

// eslint-disable-next-line no-console
console.info(
`Processed GitHub API commits history for tests/${testDirectory}`
);
} catch (error) {
console.error(
'get.commits.error',
testDirectory,
error.message
);
}
}

return knownGitCommits;
};

/**
* Determines the TestPlanVersions which can be kept or removed.
* @param testPlanVersionsByHashedTests - TestPlanVersions separated by hashedTests
* @param knownGitCommits - Ideally, the result of {@link getKnownGitCommits}
* @returns {{testPlanVersionIdsByHashedTests: {}, testPlanVersionIdsByHashedTestsToKeep: {}, testPlanVersionIdsByHashedTestsToDelete: {}}}
*/
const processTestPlanVersionIdsByHashedTests = (
testPlanVersionsByHashedTests,
knownGitCommits
) => {
for (const directory in knownGitCommits) {
const gitCommits = knownGitCommits[directory];

// Get the testPlanVersions filtered by the directory to compare against the git
// commits data
const filteredTestPlanVersionsByHashedTestsForDirectory =
Object.fromEntries(
Object.entries(testPlanVersionsByHashedTests).filter(
// eslint-disable-next-line no-unused-vars
([key, arr]) =>
arr.some(obj => obj.directory === directory)
)
);

// Sort the array for each hash object so the latest date is preferred when
// assigning the isPriority flag
for (const hash in filteredTestPlanVersionsByHashedTestsForDirectory) {
filteredTestPlanVersionsByHashedTestsForDirectory[
hash
].sort((a, b) => {
const dateA = new Date(a.updatedAt);
const dateB = new Date(b.updatedAt);
return dateB - dateA;
});
}

for (const gitCommit of gitCommits) {
const { sha } = gitCommit;
for (const hash in filteredTestPlanVersionsByHashedTestsForDirectory) {
for (const testPlanVersion of filteredTestPlanVersionsByHashedTestsForDirectory[
hash
]) {
// Check if the found commit is the same git sha
if (sha === testPlanVersion.gitSha) {
if (
!testPlanVersionsByHashedTests[hash].some(
obj => obj.isPriority
)
)
testPlanVersionsByHashedTests[hash].find(
obj => obj.id === testPlanVersion.id
).isPriority = true;
}
}
}
}
}

for (const key in testPlanVersionsByHashedTests) {
testPlanVersionsByHashedTests[key]
.sort((a, b) => {
const dateA = new Date(a.updatedAt);
const dateB = new Date(b.updatedAt);
return dateA - dateB;
})
.sort((a, b) => {
return a.isPriority ? -1 : b.isPriority ? 1 : 0;
});
}

const batchesWIsPriority = {};
const batchesWOIsPriority = {};

for (const hash in testPlanVersionsByHashedTests) {
const batch = testPlanVersionsByHashedTests[hash];

if (batch.some(testPlanVersion => testPlanVersion.isPriority)) {
batchesWIsPriority[hash] = batch;
} else {
batchesWOIsPriority[hash] = batch;
}
}

const getTestPlanVersionIdsByHashedTests = data => {
const getIdsFromKey = key => {
return data[key].map(item => item.id);
};

// Get the ids for each key
const idsByKeys = {};
Object.keys(data).forEach(key => {
idsByKeys[key] = getIdsFromKey(key);
});

return idsByKeys;
};

const testPlanVersionIdsByHashedTests =
getTestPlanVersionIdsByHashedTests(
testPlanVersionsByHashedTests
);
const testPlanVersionIdsByHashedTestsToKeep =
getTestPlanVersionIdsByHashedTests(batchesWIsPriority);
const testPlanVersionIdsByHashedTestsToDelete =
getTestPlanVersionIdsByHashedTests(batchesWOIsPriority);

return {
testPlanVersionIdsByHashedTests,
testPlanVersionIdsByHashedTestsToKeep,
testPlanVersionIdsByHashedTestsToDelete
};
};

return queryInterface.sequelize.transaction(async transaction => {
await queryInterface.addColumn(
'TestPlanVersion',
Expand All @@ -287,17 +500,32 @@ module.exports = {
);

// Get the unique TestPlanVersions found for each hash
const { testPlanVersionIdsByHashedTests } =
const { testPlanVersionsByHashedTests } =
await computeTestPlanVersionHashedTests(transaction);

const uniqueHashCount = Object.keys(
testPlanVersionIdsByHashedTests
testPlanVersionsByHashedTests
).length;
const testPlanReportsBatchSize = 100;
const iterationsNeeded = Math.ceil(
uniqueHashCount / testPlanReportsBatchSize
);

// Retrieve the latest known git commits info for each test plan directory
const knownGitCommits = await getKnownGitCommits();

const {
testPlanVersionIdsByHashedTests,
testPlanVersionIdsByHashedTestsToDelete
} = processTestPlanVersionIdsByHashedTests(
testPlanVersionsByHashedTests,
knownGitCommits
);

const testPlanVersionIdsToDelete = Object.values(
testPlanVersionIdsByHashedTestsToDelete
);

for (let i = 0; i < iterationsNeeded; i += 1) {
// eslint-disable-next-line no-console
console.info(
Expand All @@ -324,7 +552,53 @@ module.exports = {
);
}

if (uniqueHashCount) {
// Remove the TestPlanVersions not captured by removeTestPlanVersionDuplicates()
if (testPlanVersionIdsToDelete.length) {
// Update TestPlanVersion -> TestPlanReport fkey to add cascade deletion on
// TestPlanVersion row deletion
await queryInterface.sequelize.query(
`alter table public."TestPlanReport"
drop constraint "TestPlanReport_testPlan_fkey";

alter table public."TestPlanReport"
add constraint "TestPlanReport_testPlan_fkey" foreign key ("testPlanVersionId") references public."TestPlanVersion" on update cascade on delete cascade;`,
{
transaction
}
);

const toRemove = testPlanVersionIdsToDelete.flat();
await queryInterface.sequelize.query(
`DELETE FROM "TestPlanVersion" WHERE id IN (?)`,
{
replacements: [toRemove],
transaction
}
);

// Update TestPlanVersion -> TestPlanReport fkey to remove cascade delete on
// TestPlanVersion row deletion
await queryInterface.sequelize.query(
`alter table public."TestPlanReport"
drop constraint "TestPlanReport_testPlan_fkey";

alter table public."TestPlanReport"
add constraint "TestPlanReport_testPlan_fkey" foreign key ("testPlanVersionId") references public."TestPlanVersion" on update cascade;`,
{
transaction
}
);

if (uniqueHashCount) {
// eslint-disable-next-line no-console
console.info(
'Fixed',
uniqueHashCount - testPlanVersionIdsToDelete.length,
'of',
uniqueHashCount - testPlanVersionIdsToDelete.length
);
}
} else if (uniqueHashCount) {
// eslint-disable-next-line no-console
console.info('Fixed', uniqueHashCount, 'of', uniqueHashCount);
}
Expand Down
Loading