Implementing success exit codes config and proper visualization.

Author: krulis-martin

src/components/Pipelines/ParametersList/ParametersList.js

@@ -37,6 +37,12 @@ const pipelineParams = {
       defaultMessage="Extra source files can be added to tested solution"
     />
   ),
+  hasSuccessExitCodes: (
+    <FormattedMessage
+      id="app.pipelineParams.hasSuccessExitCodes"
+      defaultMessage="Configurable exit-codes that are accepted as a success when tested soution is executed"
+    />
+  ),
 };
 
 const pipelineParameterMapping = parameter => {

src/components/Solutions/TestResultsTable/TestResultsTableRow.js

@@ -6,6 +6,7 @@ import { OverlayTrigger, Tooltip } from 'react-bootstrap';
 import prettyMs from 'pretty-ms';
 
 import Button, { TheButtonGroup } from '../../widgets/TheButton';
+import Explanation from '../../widgets/Explanation';
 import { prettyPrintBytes } from '../../helpers/stringFormatters.js';
 import exitCodeMapping from '../../helpers/exitCodeMapping.js';
 import Icon, { SuccessOrFailureIcon } from '../../icons';
@@ -103,6 +104,8 @@ const TestResultsTableRow = ({
     wallTime,
     cpuTime,
     exitCode,
+    exitCodeOk,
+    exitCodeNative,
     exitSignal,
     judgeLogStdout = '',
     judgeLogStderr = '',
@@ -171,7 +174,23 @@ const TestResultsTableRow = ({
           </strong>
         </OverlayTrigger>
       )}
-      {(exitCode !== -1 || !exitSignal) && exitCodeMapping(runtimeEnvironmentId, exitCode)}
+
+      {exitCodeNative ? (
+        <>
+          <SuccessOrFailureIcon success={exitCodeOk} gapRight />
+          {exitCodeOk ? exitCode : exitCodeMapping(runtimeEnvironmentId, exitCode)}
+          {(exitCode === 0) !== exitCodeOk && (
+            <Explanation id={`exit-code-expl-${testName}`} placement="bottom">
+              <FormattedMessage
+                id="app.submissions.testResultsTable.exitCodeExplanation"
+                defaultMessage="Usually, exit code 0 is treated as success and non-zero codes as errors. This exercise have specified alternative exit codes that are treated as a success of the executed solution."
+              />
+            </Explanation>
+          )}
+        </>
+      ) : (
+        !exitSignal && status !== 'SKIPPED' && <span className="text-muted">{exitCode}</span>
+      )}
     </td>
 
     {(showJudgeLogStdout || showJudgeLogStderr) && (
@@ -240,6 +259,8 @@ TestResultsTableRow.propTypes = {
     wallTime: PropTypes.number,
     cpuTime: PropTypes.number,
     exitCode: PropTypes.number,
+    exitCodeOk: PropTypes.bool,
+    exitCodeNative: PropTypes.bool,
     exitSignal: PropTypes.number,
     judgeLogStdout: PropTypes.string,
     judgeLogStderr: PropTypes.string,

src/components/forms/EditExerciseSimpleConfigForm/EditExerciseSimpleConfigForm.js

@@ -63,6 +63,19 @@ const someValuesHaveNozeroLength = (obj, ...keys) => {
   return false;
 };
 
+const nonDefaultSuccessExitCodes = obj => {
+  if (!obj || typeof obj !== 'object' || !obj['success-exit-codes']) {
+    return false;
+  }
+
+  for (const val of Object.values(obj['success-exit-codes'])) {
+    if (val.trim() !== '0') {
+      return true;
+    }
+  }
+  return false;
+};
+
 /**
  * Make sure file(s) in form data (specified by given path) exist.
  * If not, proper form error message(s) is/are filled.
@@ -156,6 +169,7 @@ class EditExerciseSimpleConfigForm extends Component {
 
   render() {
     const {
+      initialValues,
       reset,
       change,
       handleSubmit,
@@ -251,13 +265,14 @@ class EditExerciseSimpleConfigForm extends Component {
             {exerciseTests
               .sort((a, b) => a.name.localeCompare(b.name, locale))
               .map((test, idx) => {
-                const testData = formValues && formValues.config && formValues.config[encodeNumId(test.id)];
-                const hasCompilationSetting = someValuesHaveNozeroLength(
-                  testData,
-                  'extra-files',
-                  'jar-files',
-                  'compile-args'
-                );
+                const testData = formValues?.config?.[encodeNumId(test.id)];
+                const hasCompilationSetting =
+                  someValuesHaveNozeroLength(
+                    initialValues?.config?.[encodeNumId(test.id)],
+                    'extra-files',
+                    'jar-files',
+                    'compile-args'
+                  ) || nonDefaultSuccessExitCodes(initialValues?.config?.[encodeNumId(test.id)]);
                 return (
                   <EditExerciseSimpleConfigTest
                     change={change}

src/components/forms/EditExerciseSimpleConfigForm/EditExerciseSimpleConfigTestCompilation.js

@@ -9,7 +9,7 @@ import { EMPTY_ARRAY } from '../../../helpers/common.js';
 import Button from '../../widgets/TheButton';
 import InsetPanel from '../../widgets/InsetPanel';
 import Icon, { ExpandCollapseIcon, WarningIcon } from '../../icons';
-import { SelectField, ExpandingInputFilesField, ExpandingSelectField, ExpandingTextField } from '../Fields';
+import { TextField, SelectField, ExpandingInputFilesField, ExpandingSelectField, ExpandingTextField } from '../Fields';
 import Confirm from '../../forms/Confirm';
 import Explanation from '../../widgets/Explanation';
 import {
@@ -20,6 +20,7 @@ import {
   ENV_MAVEN_ID,
   ENV_SYCL_ID,
 } from '../../../helpers/exercise/environments.js';
+import { validateExitCodes } from '../../../helpers/exercise/config.js';
 
 const COMPILER_ARGS_ENVS = [ENV_C_GCC_ID, ENV_CPP_GCC_ID, ENV_ARDUINO_ID, ENV_SYCL_ID];
 
@@ -277,6 +278,28 @@ class EditExerciseSimpleConfigTestCompilation extends Component {
                                      * End of special case for Maven
                                      */
                                   )}
+
+                                  <Field
+                                    name={`${test}.success-exit-codes.${env.id}`}
+                                    component={TextField}
+                                    disabled={readOnly}
+                                    maxLength={1024}
+                                    validate={validateExitCodes}
+                                    label={
+                                      <>
+                                        <FormattedMessage
+                                          id="app.editExerciseSimpleConfigTests.successExitCodes"
+                                          defaultMessage="Accepted exit codes:"
+                                        />
+                                        <Explanation id={`${test}.exec-targets-explanation`}>
+                                          <FormattedMessage
+                                            id="app.editExerciseSimpleConfigTests.successExitCodesExplanation"
+                                            defaultMessage="List of exit codes that are accepted as successful completion of tested solution. Exit codes are separated by commas, ranges of codes may be specifed using dash '-' (e.g., '1, 3, 5-7'). Zero is the default accepted exit code."
+                                          />
+                                        </Explanation>
+                                      </>
+                                    }
+                                  />
                                 </Col>
                               </Row>
                             </Container>

src/components/forms/EditPipelineForm/EditPipelineForm.js

@@ -161,6 +161,19 @@ class EditPipelineForm extends Component {
                     }
                   />
                 </Col>
+                <Col xl={6} lg={12}>
+                  <Field
+                    name="parameters.hasSuccessExitCodes"
+                    component={CheckboxField}
+                    onOff
+                    label={
+                      <FormattedMessage
+                        id="app.editPipelineForm.hasSuccessExitCodes"
+                        defaultMessage="Has success exit codes"
+                      />
+                    }
+                  />
+                </Col>
               </Row>
             </>
           )}

src/components/helpers/exitCodeMapping.js

@@ -12,7 +12,6 @@ import {
  * https://www.freepascal.org/docs-html/user/userap4.html
  */
 const pascalCodes = {
-  0: <FormattedMessage id="app.exitCodes.pascal.0" defaultMessage="OK" />,
   1: <FormattedMessage id="app.exitCodes.pascal.1" defaultMessage="Invalid function number" />,
   2: <FormattedMessage id="app.exitCodes.pascal.2" defaultMessage="File not found" />,
   3: <FormattedMessage id="app.exitCodes.pascal.3" defaultMessage="Path not found" />,
@@ -70,7 +69,6 @@ const pascalCodes = {
 };
 
 const python3Codes = {
-  0: <FormattedMessage id="app.exitCodes.python3.0" defaultMessage="OK" />,
   1: <FormattedMessage id="app.exitCodes.python3.1" defaultMessage="Base exception" />,
   101: <FormattedMessage id="app.exitCodes.python3.101" defaultMessage="Assertion error" />,
   102: <FormattedMessage id="app.exitCodes.python3.102" defaultMessage="Type error" />,
@@ -119,8 +117,6 @@ const exitCodeMapping = (runtimeEnvironmentId, exitCode) => {
 // TODO - eventually replace those switches with objects.
 const javaMapping = exitCode => {
   switch (exitCode) {
-    case 0:
-      return <FormattedMessage id="app.exitCodes.java.0" defaultMessage="OK" />;
     case 1:
       return <FormattedMessage id="app.exitCodes.java.1" defaultMessage="Unknown error" />;
     case 2:
@@ -160,8 +156,6 @@ const javaMapping = exitCode => {
 
 const csDotnetMapping = exitCode => {
   switch (exitCode) {
-    case 0:
-      return <FormattedMessage id="app.exitCodes.csharp.0" defaultMessage="OK" />;
     case 1:
       return <FormattedMessage id="app.exitCodes.csharp.1" defaultMessage="User error" />;
     case 101:

src/helpers/exercise/config.js

@@ -12,3 +12,88 @@ export const SUBMIT_BUTTON_MESSAGES = {
   success: <FormattedMessage id="app.editExerciseConfig.success" defaultMessage="Configuration Saved." />,
   validating: <FormattedMessage id="generic.validating" defaultMessage="Validating..." />,
 };
+
+/*
+ * Exit codes config handling
+ */
+const _validateExitCodes = value =>
+  value &&
+  value.match(/^\s*[0-9]{0,3}(-[0-9]{0,3})?(\s*,\s*[0-9]{0,3}(-[0-9]{0,3})?)*\s*$/) !== null &&
+  value.split(/[-,]/).every(val => {
+    const num = parseInt(val);
+    return !isNaN(num) && num >= 0 && num <= 255;
+  });
+
+export const validateExitCodes = value =>
+  !_validateExitCodes(value) ? (
+    <FormattedMessage
+      id="app.editExerciseConfigForm.validation.successExitCodes"
+      defaultMessage="Exit codes must be numerical values (in 0-255 range) or value intervals (written as 'from-to') separated by commas (e.g., '1, 3, 5-7')."
+    />
+  ) : undefined;
+
+const _addToBitmap = (bitmap, from, to = from) => {
+  if (to < from) {
+    const tmp = from;
+    from = to;
+    to = tmp;
+  }
+
+  while (bitmap.length < from) {
+    bitmap.push(false);
+  }
+
+  while (from <= to) {
+    if (bitmap.length > from) {
+      bitmap[from] = true;
+    } else {
+      bitmap.push(true);
+    }
+    ++from;
+  }
+};
+
+/**
+ * Convert exit codes in a string into a bitmap (array, where valid codes are true, invalid false).
+ * @param {string} str comma separated list of values and intervals
+ * @returns {Array}
+ */
+export const exitCodesStrToBitmap = str => {
+  const res = [];
+  if (_validateExitCodes(str)) {
+    str
+      .split(',')
+      .map(val => val.trim())
+      .forEach(val => {
+        _addToBitmap(res, ...val.split('-')); // split will produce either one or two values
+      });
+  }
+  return res;
+};
+
+/**
+ * Convert exit codes bitmap into normalized array of tokens (each token is either a value or an interval)
+ * @param {Array} bitmap
+ * @returns {Array}
+ */
+export const exitCodesBitmapToTokens = bitmap => {
+  const tokens = [];
+  let i = 0;
+  while (i < bitmap.length) {
+    while (i < bitmap.length && !bitmap[i]) ++i;
+    if (i < bitmap.length) {
+      const from = i;
+      while (i < bitmap.length && bitmap[i]) ++i;
+      const to = i - 1;
+      tokens.push(from === to ? from : `${from}-${to}`);
+    }
+  }
+  return tokens;
+};
+
+/**
+ * Convert exit codes bitmap into normalized string.
+ * @param {Array} bitmap
+ * @returns {string} comma separated list of values and intervals
+ */
+export const exitCodesBitmapToStr = bitmap => exitCodesBitmapToTokens(bitmap).join(', ');

src/helpers/exercise/configSimple.js

@@ -13,6 +13,7 @@ import {
   ENV_SCALA_ID,
   ENV_SYCL_ID,
 } from './environments.js';
+import { exitCodesStrToBitmap, exitCodesBitmapToTokens } from './config.js';
 
 /**
  * Base class for all pipeline variables being edited in the config form.
@@ -316,6 +317,14 @@ const _PIPELINE_DEFAULT_VARS_DESCRIPTORS = [
     }))
     .setTransformPostprocess(value => value || '$entry-point')
     .forCompilationAndExecution(),
+  new Variable('success-exit-codes', 'string[]', '0')
+    .individualEnvs()
+    .setPipelineFilter('hasSuccessExitCodes')
+    .setInitialPostprocess(({ 'success-exit-codes': codes }) => ({
+      'success-exit-codes': objectMap(codes, c => (c || ['0']).join(', ')),
+    }))
+    .setTransformPostprocess(codes => exitCodesBitmapToTokens(exitCodesStrToBitmap(codes)))
+    .forCompilationAndExecution(),
 ];
 
 const _ENV_SPECIFIC_VARS_DESCRIPTORS = {