Update results collection form to support pass/fail assertion verdicts

lib/data/process-test-directory-v1.js

@@ -137,10 +137,10 @@ const processTestDirectory = async ({ directory, args = {} }) => {
   });
 
   const keyDefs = {};
-  const support = JSON.parse(supportRecord.text);
+  const supportJson = JSON.parse(supportRecord.text);
 
-  let allATKeys = support.ats.map(({ key }) => key);
-  let allATNames = support.ats.map(({ name }) => name);
+  let allATKeys = supportJson.ats.map(({ key }) => key);
+  let allATNames = supportJson.ats.map(({ name }) => name);
 
   const validAppliesTo = ['Screen Readers', 'Desktop Screen Readers'].concat(allATKeys);
 
@@ -414,6 +414,7 @@ const processTestDirectory = async ({ directory, args = {} }) => {
       applies_to: appliesTo,
       mode: mode,
       task: task,
+      assertionResponseQuestion: supportJson.testPlanStrings.assertionResponseQuestion,
       specific_user_instruction: test.instructions,
       output_assertions: assertions,
     };
@@ -714,7 +715,7 @@ ${rows}
   const testsParsed = tests.map(parseTestCSVRow);
   const referencesParsed = parseReferencesCSV(refRows);
   const keysParsed = parseKeyMap(keyDefs);
-  const supportParsed = parseSupport(support);
+  const supportParsed = parseSupport(supportJson);
 
   const keysValidated = validateKeyMap(keysParsed, {
     addKeyMapError(reason) {

lib/data/process-test-directory.js

@@ -528,6 +528,7 @@ const processTestDirectory = async ({ directory, args = {} }) => {
       setup_script_description: getSetupScriptDescription(test.setupScript.scriptDescription),
       setupTestPage: test.setupScript.script,
       specific_user_instruction: test.instructions,
+      assertionResponseQuestion: supportJson.testPlanStrings.assertionResponseQuestion,
       commandsInfo: test.commandsInfo,
       output_assertions: test.assertions,
     };

tests/resources/aria-at-harness.mjs

@@ -8,7 +8,12 @@ import {
   focus,
   render,
 } from './vrender.mjs';
-import { userCloseWindow, userOpenWindow, WhitespaceStyleMap } from './aria-at-test-run.mjs';
+import {
+  AssertionResultMap,
+  userCloseWindow,
+  userOpenWindow,
+  WhitespaceStyleMap,
+} from './aria-at-test-run.mjs';
 import { TestRunExport, TestRunInputOutput } from './aria-at-test-io-format.mjs';
 import { TestWindow } from './aria-at-test-window.mjs';
 
@@ -39,6 +44,14 @@ const PAGE_STYLES = `
    margin-left: 1em;
   }
 
+  fieldset.assertions {
+    margin-bottom: 1em;
+  }
+
+  label.assertion {
+    display: block;
+  }
+
   .required:not(.highlight-required) {
     display: none;
   }
@@ -382,12 +395,9 @@ function renderVirtualInstructionDocument(doc) {
           )
         )
       ),
-      table(
-        tr(
-          th(rich(command.assertionsHeader.descriptionHeader)),
-          th(rich(command.assertionsHeader.passHeader)),
-          th(rich(command.assertionsHeader.failHeader))
-        ),
+      fieldset(
+        className(['assertions']),
+        legend(rich(command.assertionsHeader.descriptionHeader)),
         ...command.assertions.map(bind(commandResultAssertion, commandIndex))
       ),
       ...[command.unexpectedBehaviors].map(bind(commandResultUnexpectedBehavior, commandIndex))
@@ -479,26 +489,15 @@ function renderVirtualInstructionDocument(doc) {
    * @param {number} assertionIndex
    */
   function commandResultAssertion(commandIndex, assertion, assertionIndex) {
-    return tr(
-      td(rich(assertion.description)),
-      td(
-        ...[assertion.passChoice].map(choice =>
-          radioChoice(
-            `pass-${commandIndex}-${assertionIndex}`,
-            `result-${commandIndex}-${assertionIndex}`,
-            choice
-          )
-        )
+    return label(
+      className(['assertion']),
+      input(
+        type('checkbox'),
+        id(`cmd-${commandIndex}-${assertionIndex}`),
+        checked(assertion.passed === AssertionResultMap.PASS),
+        onclick(assertion.click)
       ),
-      td(
-        ...assertion.failChoices.map((choice, failIndex) =>
-          radioChoice(
-            `${failIndex === 0 ? 'missing' : 'fail'}-${commandIndex}-${assertionIndex}`,
-            `result-${commandIndex}-${assertionIndex}`,
-            choice
-          )
-        )
-      )
+      rich(assertion.description)
     );
   }
 

tests/resources/aria-at-test-io-format.mjs

@@ -760,6 +760,7 @@ class BehaviorInput {
         specificUserInstruction: json.specific_user_instruction,
         setupScriptDescription: json.setup_script_description,
         setupTestPage: json.setupTestPage,
+        assertionResponseQuestion: json.assertionResponseQuestion,
         commands: commandsAndSettings,
         assertions: (json.output_assertions ? json.output_assertions : []).map(assertion => {
           // Tuple array [ priorityNumber, assertionText ]
@@ -1159,6 +1160,7 @@ export class TestRunInputOutput {
       openTest: {
         enabled: true,
       },
+      assertionResponseQuestion: test.assertionResponseQuestion,
       commands: test.commands.map(
         command =>
           /** @type {import("./aria-at-test-run.mjs").TestRunCommand} */ ({
@@ -1451,7 +1453,7 @@ export class TestRunInputOutput {
                 ? 'failIncorrect'
                 : assertionResult.failedReason === 'NO_OUTPUT'
                 ? 'failMissing'
-                : 'notSet',
+                : 'fail',
             };
           }),
           unexpected: {
@@ -1532,6 +1534,7 @@ export class TestRunExport extends TestRun {
 
 const AssertionPassJSONMap = createEnumMap({
   GOOD_OUTPUT: 'Good Output',
+  PASS: 'Pass',
 });
 
 /**
@@ -1553,6 +1556,7 @@ const AssertionFailJSONMap = createEnumMap({
   NO_OUTPUT: 'No Output',
   INCORRECT_OUTPUT: 'Incorrect Output',
   NO_SUPPORT: 'No Support',
+  FAIL: 'Fail',
 });
 
 /** @typedef {SubmitResultDetailsCommandsAssertionsPass | SubmitResultDetailsCommandsAssertionsFail} SubmitResultAssertionsJSON */

tests/resources/aria-at-test-run.mjs

@@ -209,24 +209,6 @@ export function instructionDocument(resultState, hooks) {
       : null,
   };
 
-  /**
-   * @param {T} resultAssertion
-   * @param {T["result"]} resultValue
-   * @param {Omit<InstructionDocumentAssertionChoice, 'checked' | 'focus'>} partialChoice
-   * @returns {InstructionDocumentAssertionChoice}
-   * @template {TestRunAssertion | TestRunAdditionalAssertion} T
-   */
-  function assertionChoice(resultAssertion, resultValue, partialChoice) {
-    return {
-      ...partialChoice,
-      checked: resultAssertion.result === resultValue,
-      focus:
-        resultState.currentUserAction === 'validateResults' &&
-        resultAssertion.highlightRequired &&
-        focusFirstRequired(),
-    };
-  }
-
   /**
    * @param {string} command
    * @param {number} commandIndex
@@ -261,9 +243,9 @@ export function instructionDocument(resultState, hooks) {
         change: atOutput => hooks.setCommandOutput({ commandIndex, atOutput }),
       },
       assertionsHeader: {
-        descriptionHeader: 'Assertion',
-        passHeader: 'Success case',
-        failHeader: 'Failure cases',
+        descriptionHeader: `${resultState.assertionResponseQuestion} ${command}${
+          settingsText && settings !== 'defaultMode' ? ` (${settingsText})` : ''
+        }?`,
       },
       assertions: [
         ...assertions.map(bind(assertionResult, commandIndex)),
@@ -382,57 +364,17 @@ export function instructionDocument(resultState, hooks) {
   function assertionResult(commandIndex, assertion, assertionIndex) {
     const resultAssertion = resultState.commands[commandIndex].assertions[assertionIndex];
     return /** @type {InstructionDocumentResultsCommandsAssertion} */ ({
-      description: [
-        assertion,
-        {
-          required: true,
-          highlightRequired: resultAssertion.highlightRequired,
-          description: '(required: mark output)',
-        },
-      ],
-      passChoice: assertionChoice(resultAssertion, CommonResultMap.PASS, {
-        label: [
-          `Good Output `,
-          {
-            offScreen: true,
-            description: 'for assertion',
-          },
-        ],
-        click: () =>
-          hooks.setCommandAssertion({ commandIndex, assertionIndex, result: CommonResultMap.PASS }),
-      }),
-      failChoices: [
-        assertionChoice(resultAssertion, AssertionResultMap.FAIL_MISSING, {
-          label: [
-            `No Output `,
-            {
-              offScreen: true,
-              description: 'for assertion',
-            },
-          ],
-          click: () =>
-            hooks.setCommandAssertion({
-              commandIndex,
-              assertionIndex,
-              result: AssertionResultMap.FAIL_MISSING,
-            }),
+      description: [assertion],
+      passed: resultAssertion.result === AssertionResultMap.PASS,
+      click: () =>
+        hooks.setCommandAssertion({
+          commandIndex,
+          assertionIndex,
+          result:
+            resultAssertion.result === AssertionResultMap.PASS
+              ? AssertionResultMap.FAIL
+              : AssertionResultMap.PASS,
         }),
-        assertionChoice(resultAssertion, AssertionResultMap.FAIL_INCORRECT, {
-          label: [
-            `Incorrect Output `,
-            {
-              offScreen: true,
-              description: 'for assertion',
-            },
-          ],
-          click: () =>
-            hooks.setCommandAssertion({
-              commandIndex,
-              assertionIndex,
-              result: AssertionResultMap.FAIL_INCORRECT,
-            }),
-        }),
-      ],
     });
   }
 
@@ -445,34 +387,17 @@ export function instructionDocument(resultState, hooks) {
     const resultAdditionalAssertion =
       resultState.commands[commandIndex].additionalAssertions[assertionIndex];
     return /** @type {InstructionDocumentResultsCommandsAssertion} */ ({
-      description: [
-        assertion,
-        {
-          required: true,
-          highlightRequired: resultAdditionalAssertion.highlightRequired,
-          description: '(required: mark support)',
-        },
-      ],
-      passChoice: assertionChoice(resultAdditionalAssertion, AdditionalAssertionResultMap.PASS, {
-        label: ['Good Support ', { offScreen: true, description: 'for assertion' }],
-        click: () =>
-          hooks.setCommandAdditionalAssertion({
-            commandIndex,
-            additionalAssertionIndex: assertionIndex,
-            result: AdditionalAssertionResultMap.PASS,
-          }),
-      }),
-      failChoices: [
-        assertionChoice(resultAdditionalAssertion, AdditionalAssertionResultMap.FAIL_SUPPORT, {
-          label: ['No Support ', { offScreen: true, description: 'for assertion' }],
-          click: () =>
-            hooks.setCommandAdditionalAssertion({
-              commandIndex,
-              additionalAssertionIndex: assertionIndex,
-              result: AdditionalAssertionResultMap.FAIL_SUPPORT,
-            }),
+      description: [assertion],
+      passed: resultAdditionalAssertion.result === CommonResultMap.PASS,
+      click: () =>
+        hooks.setCommandAssertion({
+          commandIndex,
+          assertionIndex,
+          result:
+            resultAdditionalAssertion.result === AssertionResultMap.PASS
+              ? AssertionResultMap.FAIL
+              : AssertionResultMap.PASS,
         }),
-      ],
     });
   }
 }
@@ -535,6 +460,7 @@ export const AssertionResultMap = createEnumMap({
   ...CommonResultMap,
   FAIL_MISSING: 'failMissing',
   FAIL_INCORRECT: 'failIncorrect',
+  FAIL: 'fail',
 });
 
 /**
@@ -796,10 +722,6 @@ function isSomeFieldRequired(state) {
   return state.commands.some(
     command =>
       command.atOutput.value.trim() === '' ||
-      command.assertions.some(assertion => assertion.result === CommonResultMap.NOT_SET) ||
-      command.additionalAssertions.some(
-        assertion => assertion.result === CommonResultMap.NOT_SET
-      ) ||
       command.unexpected.hasUnexpected === HasUnexpectedBehaviorMap.NOT_SET ||
       (command.unexpected.hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED &&
         (command.unexpected.behaviors.every(({ checked }) => !checked) ||
@@ -975,14 +897,6 @@ export function userValidateState() {
             ...command.atOutput,
             highlightRequired: !command.atOutput.value.trim(),
           },
-          assertions: command.assertions.map(assertion => ({
-            ...assertion,
-            highlightRequired: assertion.result === CommonResultMap.NOT_SET,
-          })),
-          additionalAssertions: command.additionalAssertions.map(assertion => ({
-            ...assertion,
-            highlightRequired: assertion.result === CommonResultMap.NOT_SET,
-          })),
           unexpected: {
             ...command.unexpected,
             highlightRequired:
@@ -1100,15 +1014,14 @@ export function userValidateState() {
 /**
  * @typedef InstructionDocumentResultsCommandsAssertion
  * @property {Description} description
- * @property {InstructionDocumentAssertionChoice} passChoice
- * @property {InstructionDocumentAssertionChoice[]} failChoices
+ * @property {Boolean} passed
+ * @property {boolean} [focus]
+ * @property {() => void} click
  */
 
 /**
  * @typedef InstructionDocumentResultsCommandsAssertionsHeader
  * @property {Description} descriptionHeader
- * @property {Description} passHeader
- * @property {Description} failHeader
  */
 
 /**

types/aria-at-file.js

@@ -51,6 +51,7 @@
  * @typedef AriaATFile.Behavior
  * @property {string} setup_script_description
  * @property {string} setupTestPage
+ * @property {string} assertionResponseQuestion
  * @property {string[]} applies_to
  * @property {AriaATFile.ATMode | AriaATFile.ATMode[]} mode
  * @property {string} task