From 7d068bd5a761550810a69ee0b0abee8a33134fa1 Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Wed, 8 Nov 2023 12:06:27 -0800 Subject: [PATCH 1/5] Initial rendering pass --- tests/resources/aria-at-harness.mjs | 43 ++++++++++++---------------- tests/resources/aria-at-test-run.mjs | 6 ++-- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/tests/resources/aria-at-harness.mjs b/tests/resources/aria-at-harness.mjs index 4ef221733..2aa11acb5 100644 --- a/tests/resources/aria-at-harness.mjs +++ b/tests/resources/aria-at-harness.mjs @@ -39,6 +39,14 @@ const PAGE_STYLES = ` margin-left: 1em; } + fieldset.assertions { + margin-bottom: 1em; + } + + label.assertion { + display: block; + } + .required:not(.highlight-required) { display: none; } @@ -382,12 +390,9 @@ function renderVirtualInstructionDocument(doc) { ) ) ), - table( - tr( - th(rich(command.assertionsHeader.descriptionHeader)), - th(rich(command.assertionsHeader.passHeader)), - th(rich(command.assertionsHeader.failHeader)) - ), + fieldset( + className(['assertions']), + legend(rich(command.assertionsHeader.descriptionHeader)), ...command.assertions.map(bind(commandResultAssertion, commandIndex)) ), ...[command.unexpectedBehaviors].map(bind(commandResultUnexpectedBehavior, commandIndex)) @@ -479,26 +484,14 @@ function renderVirtualInstructionDocument(doc) { * @param {number} assertionIndex */ function commandResultAssertion(commandIndex, assertion, assertionIndex) { - return tr( - td(rich(assertion.description)), - td( - ...[assertion.passChoice].map(choice => - radioChoice( - `pass-${commandIndex}-${assertionIndex}`, - `result-${commandIndex}-${assertionIndex}`, - choice - ) - ) + return label( + className(['assertion']), + input( + type('checkbox'), + id(`cmd-${commandIndex}-${assertionIndex}`), + checked(assertion.choice !== undefined) ), - td( - ...assertion.failChoices.map((choice, failIndex) => - radioChoice( - `${failIndex === 0 ? 'missing' : 'fail'}-${commandIndex}-${assertionIndex}`, - `result-${commandIndex}-${assertionIndex}`, - choice - ) - ) - ) + rich(assertion.description) ); } diff --git a/tests/resources/aria-at-test-run.mjs b/tests/resources/aria-at-test-run.mjs index b47125035..2b4ab5bb7 100644 --- a/tests/resources/aria-at-test-run.mjs +++ b/tests/resources/aria-at-test-run.mjs @@ -261,7 +261,9 @@ export function instructionDocument(resultState, hooks) { change: atOutput => hooks.setCommandOutput({ commandIndex, atOutput }), }, assertionsHeader: { - descriptionHeader: 'Assertion', + descriptionHeader: `Which statements are true about the response to '${command}'${ + settingsText && settings !== 'defaultMode' ? ` (${settingsText})` : '' + }?`, passHeader: 'Success case', failHeader: 'Failure cases', }, @@ -1107,8 +1109,6 @@ export function userValidateState() { /** * @typedef InstructionDocumentResultsCommandsAssertionsHeader * @property {Description} descriptionHeader - * @property {Description} passHeader - * @property {Description} failHeader */ /** From dc31cff6a2b58e06b78432b7d0c787fc32ea866f Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Wed, 8 Nov 2023 12:30:16 -0800 Subject: [PATCH 2/5] Initial interaction with assertion checkbox --- tests/resources/aria-at-harness.mjs | 9 ++- tests/resources/aria-at-test-run.mjs | 103 +++++++-------------------- 2 files changed, 31 insertions(+), 81 deletions(-) diff --git a/tests/resources/aria-at-harness.mjs b/tests/resources/aria-at-harness.mjs index 2aa11acb5..e9ca42267 100644 --- a/tests/resources/aria-at-harness.mjs +++ b/tests/resources/aria-at-harness.mjs @@ -8,7 +8,12 @@ import { focus, render, } from './vrender.mjs'; -import { userCloseWindow, userOpenWindow, WhitespaceStyleMap } from './aria-at-test-run.mjs'; +import { + AssertionResultMap, + userCloseWindow, + userOpenWindow, + WhitespaceStyleMap, +} from './aria-at-test-run.mjs'; import { TestRunExport, TestRunInputOutput } from './aria-at-test-io-format.mjs'; import { TestWindow } from './aria-at-test-window.mjs'; @@ -489,7 +494,7 @@ function renderVirtualInstructionDocument(doc) { input( type('checkbox'), id(`cmd-${commandIndex}-${assertionIndex}`), - checked(assertion.choice !== undefined) + checked(assertion.passed === AssertionResultMap.PASS) ), rich(assertion.description) ); diff --git a/tests/resources/aria-at-test-run.mjs b/tests/resources/aria-at-test-run.mjs index 2b4ab5bb7..4af3cb1a4 100644 --- a/tests/resources/aria-at-test-run.mjs +++ b/tests/resources/aria-at-test-run.mjs @@ -384,57 +384,17 @@ export function instructionDocument(resultState, hooks) { function assertionResult(commandIndex, assertion, assertionIndex) { const resultAssertion = resultState.commands[commandIndex].assertions[assertionIndex]; return /** @type {InstructionDocumentResultsCommandsAssertion} */ ({ - description: [ - assertion, - { - required: true, - highlightRequired: resultAssertion.highlightRequired, - description: '(required: mark output)', - }, - ], - passChoice: assertionChoice(resultAssertion, CommonResultMap.PASS, { - label: [ - `Good Output `, - { - offScreen: true, - description: 'for assertion', - }, - ], - click: () => - hooks.setCommandAssertion({ commandIndex, assertionIndex, result: CommonResultMap.PASS }), - }), - failChoices: [ - assertionChoice(resultAssertion, AssertionResultMap.FAIL_MISSING, { - label: [ - `No Output `, - { - offScreen: true, - description: 'for assertion', - }, - ], - click: () => - hooks.setCommandAssertion({ - commandIndex, - assertionIndex, - result: AssertionResultMap.FAIL_MISSING, - }), + description: [assertion], + passed: resultAssertion.result === AssertionResultMap.PASS, + click: () => + hooks.setCommandAssertion({ + commandIndex, + assertionIndex, + result: + resultAssertion.result === AssertionResultMap.PASS + ? AssertionResultMap.FAIL + : AssertionResultMap.PASS, }), - assertionChoice(resultAssertion, AssertionResultMap.FAIL_INCORRECT, { - label: [ - `Incorrect Output `, - { - offScreen: true, - description: 'for assertion', - }, - ], - click: () => - hooks.setCommandAssertion({ - commandIndex, - assertionIndex, - result: AssertionResultMap.FAIL_INCORRECT, - }), - }), - ], }); } @@ -447,34 +407,17 @@ export function instructionDocument(resultState, hooks) { const resultAdditionalAssertion = resultState.commands[commandIndex].additionalAssertions[assertionIndex]; return /** @type {InstructionDocumentResultsCommandsAssertion} */ ({ - description: [ - assertion, - { - required: true, - highlightRequired: resultAdditionalAssertion.highlightRequired, - description: '(required: mark support)', - }, - ], - passChoice: assertionChoice(resultAdditionalAssertion, AdditionalAssertionResultMap.PASS, { - label: ['Good Support ', { offScreen: true, description: 'for assertion' }], - click: () => - hooks.setCommandAdditionalAssertion({ - commandIndex, - additionalAssertionIndex: assertionIndex, - result: AdditionalAssertionResultMap.PASS, - }), - }), - failChoices: [ - assertionChoice(resultAdditionalAssertion, AdditionalAssertionResultMap.FAIL_SUPPORT, { - label: ['No Support ', { offScreen: true, description: 'for assertion' }], - click: () => - hooks.setCommandAdditionalAssertion({ - commandIndex, - additionalAssertionIndex: assertionIndex, - result: AdditionalAssertionResultMap.FAIL_SUPPORT, - }), + description: [assertion], + passed: resultAdditionalAssertion.result === CommonResultMap.PASS, + click: () => + hooks.setCommandAssertion({ + commandIndex, + assertionIndex, + result: + resultAdditionalAssertion.result === AssertionResultMap.PASS + ? AssertionResultMap.FAIL + : AssertionResultMap.PASS, }), - ], }); } } @@ -537,6 +480,7 @@ export const AssertionResultMap = createEnumMap({ ...CommonResultMap, FAIL_MISSING: 'failMissing', FAIL_INCORRECT: 'failIncorrect', + FAIL: 'fail', }); /** @@ -1102,8 +1046,9 @@ export function userValidateState() { /** * @typedef InstructionDocumentResultsCommandsAssertion * @property {Description} description - * @property {InstructionDocumentAssertionChoice} passChoice - * @property {InstructionDocumentAssertionChoice[]} failChoices + * @property {Boolean} passed + * @property {boolean} [focus] + * @property {() => void} click */ /** From 6a9f19209cb10cf9165737e5e86c2acac7041389 Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Wed, 8 Nov 2023 15:09:51 -0800 Subject: [PATCH 3/5] Cleanup and use assertionResponseQuestion --- lib/data/process-test-directory.js | 1 + tests/resources/aria-at-test-io-format.mjs | 6 ++++- tests/resources/aria-at-test-run.mjs | 30 +++------------------- types/aria-at-file.js | 1 + 4 files changed, 10 insertions(+), 28 deletions(-) diff --git a/lib/data/process-test-directory.js b/lib/data/process-test-directory.js index a8c9d1743..d152e4b92 100644 --- a/lib/data/process-test-directory.js +++ b/lib/data/process-test-directory.js @@ -528,6 +528,7 @@ const processTestDirectory = async ({ directory, args = {} }) => { setup_script_description: getSetupScriptDescription(test.setupScript.scriptDescription), setupTestPage: test.setupScript.script, specific_user_instruction: test.instructions, + assertionResponseQuestion: supportJson.testPlanStrings.assertionResponseQuestion, commandsInfo: test.commandsInfo, output_assertions: test.assertions, }; diff --git a/tests/resources/aria-at-test-io-format.mjs b/tests/resources/aria-at-test-io-format.mjs index ec9ca27d6..991b6e43b 100644 --- a/tests/resources/aria-at-test-io-format.mjs +++ b/tests/resources/aria-at-test-io-format.mjs @@ -760,6 +760,7 @@ class BehaviorInput { specificUserInstruction: json.specific_user_instruction, setupScriptDescription: json.setup_script_description, setupTestPage: json.setupTestPage, + assertionResponseQuestion: json.assertionResponseQuestion, commands: commandsAndSettings, assertions: (json.output_assertions ? json.output_assertions : []).map(assertion => { // Tuple array [ priorityNumber, assertionText ] @@ -1159,6 +1160,7 @@ export class TestRunInputOutput { openTest: { enabled: true, }, + assertionResponseQuestion: test.assertionResponseQuestion, commands: test.commands.map( command => /** @type {import("./aria-at-test-run.mjs").TestRunCommand} */ ({ @@ -1451,7 +1453,7 @@ export class TestRunInputOutput { ? 'failIncorrect' : assertionResult.failedReason === 'NO_OUTPUT' ? 'failMissing' - : 'notSet', + : 'fail', }; }), unexpected: { @@ -1532,6 +1534,7 @@ export class TestRunExport extends TestRun { const AssertionPassJSONMap = createEnumMap({ GOOD_OUTPUT: 'Good Output', + PASS: 'Pass', }); /** @@ -1553,6 +1556,7 @@ const AssertionFailJSONMap = createEnumMap({ NO_OUTPUT: 'No Output', INCORRECT_OUTPUT: 'Incorrect Output', NO_SUPPORT: 'No Support', + FAIL: 'Fail', }); /** @typedef {SubmitResultDetailsCommandsAssertionsPass | SubmitResultDetailsCommandsAssertionsFail} SubmitResultAssertionsJSON */ diff --git a/tests/resources/aria-at-test-run.mjs b/tests/resources/aria-at-test-run.mjs index 4af3cb1a4..0a3080d51 100644 --- a/tests/resources/aria-at-test-run.mjs +++ b/tests/resources/aria-at-test-run.mjs @@ -209,24 +209,6 @@ export function instructionDocument(resultState, hooks) { : null, }; - /** - * @param {T} resultAssertion - * @param {T["result"]} resultValue - * @param {Omit} partialChoice - * @returns {InstructionDocumentAssertionChoice} - * @template {TestRunAssertion | TestRunAdditionalAssertion} T - */ - function assertionChoice(resultAssertion, resultValue, partialChoice) { - return { - ...partialChoice, - checked: resultAssertion.result === resultValue, - focus: - resultState.currentUserAction === 'validateResults' && - resultAssertion.highlightRequired && - focusFirstRequired(), - }; - } - /** * @param {string} command * @param {number} commandIndex @@ -261,11 +243,9 @@ export function instructionDocument(resultState, hooks) { change: atOutput => hooks.setCommandOutput({ commandIndex, atOutput }), }, assertionsHeader: { - descriptionHeader: `Which statements are true about the response to '${command}'${ - settingsText && settings !== 'defaultMode' ? ` (${settingsText})` : '' - }?`, - passHeader: 'Success case', - failHeader: 'Failure cases', + descriptionHeader: `${ + resultState.assertionResponseQuestion ?? 'Which statements are true about the response to' + } ${command}${settingsText && settings !== 'defaultMode' ? ` (${settingsText})` : ''}?`, }, assertions: [ ...assertions.map(bind(assertionResult, commandIndex)), @@ -742,10 +722,6 @@ function isSomeFieldRequired(state) { return state.commands.some( command => command.atOutput.value.trim() === '' || - command.assertions.some(assertion => assertion.result === CommonResultMap.NOT_SET) || - command.additionalAssertions.some( - assertion => assertion.result === CommonResultMap.NOT_SET - ) || command.unexpected.hasUnexpected === HasUnexpectedBehaviorMap.NOT_SET || (command.unexpected.hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED && (command.unexpected.behaviors.every(({ checked }) => !checked) || diff --git a/types/aria-at-file.js b/types/aria-at-file.js index 929093d07..308dfc2dd 100644 --- a/types/aria-at-file.js +++ b/types/aria-at-file.js @@ -51,6 +51,7 @@ * @typedef AriaATFile.Behavior * @property {string} setup_script_description * @property {string} setupTestPage + * @property {string} assertionResponseQuestion * @property {string[]} applies_to * @property {AriaATFile.ATMode | AriaATFile.ATMode[]} mode * @property {string} task From 5cd09a12cd16e233ec58e89fecc87a3556ed9a67 Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Thu, 9 Nov 2023 11:21:31 -0800 Subject: [PATCH 4/5] Remove unused required validation, Ensure input fires state change event --- tests/resources/aria-at-harness.mjs | 3 ++- tests/resources/aria-at-test-run.mjs | 8 -------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/resources/aria-at-harness.mjs b/tests/resources/aria-at-harness.mjs index e9ca42267..769547871 100644 --- a/tests/resources/aria-at-harness.mjs +++ b/tests/resources/aria-at-harness.mjs @@ -494,7 +494,8 @@ function renderVirtualInstructionDocument(doc) { input( type('checkbox'), id(`cmd-${commandIndex}-${assertionIndex}`), - checked(assertion.passed === AssertionResultMap.PASS) + checked(assertion.passed === AssertionResultMap.PASS), + onclick(assertion.click) ), rich(assertion.description) ); diff --git a/tests/resources/aria-at-test-run.mjs b/tests/resources/aria-at-test-run.mjs index 0a3080d51..d5ce36bc6 100644 --- a/tests/resources/aria-at-test-run.mjs +++ b/tests/resources/aria-at-test-run.mjs @@ -897,14 +897,6 @@ export function userValidateState() { ...command.atOutput, highlightRequired: !command.atOutput.value.trim(), }, - assertions: command.assertions.map(assertion => ({ - ...assertion, - highlightRequired: assertion.result === CommonResultMap.NOT_SET, - })), - additionalAssertions: command.additionalAssertions.map(assertion => ({ - ...assertion, - highlightRequired: assertion.result === CommonResultMap.NOT_SET, - })), unexpected: { ...command.unexpected, highlightRequired: From c369edd311166a74728b77bbb10d7e31ce45af4f Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Wed, 15 Nov 2023 09:08:30 -0800 Subject: [PATCH 5/5] Use assertionResponseQuestion in v1 tests as well --- lib/data/process-test-directory-v1.js | 9 +++++---- tests/resources/aria-at-test-run.mjs | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/data/process-test-directory-v1.js b/lib/data/process-test-directory-v1.js index 434bfdc6a..c286e2c78 100644 --- a/lib/data/process-test-directory-v1.js +++ b/lib/data/process-test-directory-v1.js @@ -137,10 +137,10 @@ const processTestDirectory = async ({ directory, args = {} }) => { }); const keyDefs = {}; - const support = JSON.parse(supportRecord.text); + const supportJson = JSON.parse(supportRecord.text); - let allATKeys = support.ats.map(({ key }) => key); - let allATNames = support.ats.map(({ name }) => name); + let allATKeys = supportJson.ats.map(({ key }) => key); + let allATNames = supportJson.ats.map(({ name }) => name); const validAppliesTo = ['Screen Readers', 'Desktop Screen Readers'].concat(allATKeys); @@ -414,6 +414,7 @@ const processTestDirectory = async ({ directory, args = {} }) => { applies_to: appliesTo, mode: mode, task: task, + assertionResponseQuestion: supportJson.testPlanStrings.assertionResponseQuestion, specific_user_instruction: test.instructions, output_assertions: assertions, }; @@ -714,7 +715,7 @@ ${rows} const testsParsed = tests.map(parseTestCSVRow); const referencesParsed = parseReferencesCSV(refRows); const keysParsed = parseKeyMap(keyDefs); - const supportParsed = parseSupport(support); + const supportParsed = parseSupport(supportJson); const keysValidated = validateKeyMap(keysParsed, { addKeyMapError(reason) { diff --git a/tests/resources/aria-at-test-run.mjs b/tests/resources/aria-at-test-run.mjs index d5ce36bc6..04c7fd4c4 100644 --- a/tests/resources/aria-at-test-run.mjs +++ b/tests/resources/aria-at-test-run.mjs @@ -243,9 +243,9 @@ export function instructionDocument(resultState, hooks) { change: atOutput => hooks.setCommandOutput({ commandIndex, atOutput }), }, assertionsHeader: { - descriptionHeader: `${ - resultState.assertionResponseQuestion ?? 'Which statements are true about the response to' - } ${command}${settingsText && settings !== 'defaultMode' ? ` (${settingsText})` : ''}?`, + descriptionHeader: `${resultState.assertionResponseQuestion} ${command}${ + settingsText && settings !== 'defaultMode' ? ` (${settingsText})` : '' + }?`, }, assertions: [ ...assertions.map(bind(assertionResult, commandIndex)),