WIP: Added Shrink/Retry

AVAVerify.js

@@ -1,6 +1,8 @@
 const AVAVerifyConfig = require("./config");
 const jsverify = require("jsverify");
 const assert = require("assert");
+const FailedTestError = require("./lib/FailedTestError");
+const TestDetails = require("./lib/TestDetails");
 
 let unnamedSuites = 0;
 
@@ -59,16 +61,60 @@ class AVAVerify {
     };
   }
 
+  /**
+   * Shrink and retry a failing test.
+   * @param {Test} t the `Test` object from an AVA test
+   * @param {TestDetails} details the details about the current test.
+   * @return {Promise} resolves when testing is complete.
+   * @todo Implement.
+  */
+  retryBody(t, details) {
+
+  }
+
   /**
    * The contents to use inside an AVA `test` block to run a test instance.
    * @param {Number} i the test index
-   * @todo Implement.
+   * @todo Publicize `TestDetails` object being in `t.context` (warn that it is unstable)
    * @return {Function} the `test` block contents.
   */
   testBody(i) {
     return (t) => {
-      return this.constructor.genArbs(this.opts.size, this.arbs)
-        .then(vals => this.body(t, ...vals));
+      const details = new TestDetails({
+        verify: this,
+        index: i,
+      });
+      t.context._avaVerify = details;
+      return this.constructor
+        .genArbs(this.opts.size, this.arbs)
+        .then(vals => details.storeFirstValues(vals))
+        .then(vals => this.body(t, ...vals))
+        .then(() => {
+          if(t.assertError || t.calledEnd) {
+            throw new FailedTestError();
+          }
+          // TODO: broadcast test success
+        })
+        .catch(err => {
+          details.failed = true;
+          details.failureDetails = this.constructor.getFailureDetails(t);
+          // TODO: catch any other test failures?  err.name === AssertionError?
+          const retriable = err instanceof FailedTestError;
+          if(details.shrink && retriable) {
+            return this
+              .retryBody(t, details)
+              .then(() => this.constructor.restoreFailureDetails(t, details.failureDetails))
+              .then(() => {
+                //TODO: broadcast test finish
+                //TODO: if test rejected above, reject again?
+                return true;
+              });
+          }
+          else {
+            //TODO: broadcast test finish
+            throw err;
+          }
+        });
     };
   }
 

lib/ExtendableError.js

@@ -0,0 +1,17 @@
+/**
+ * From https://stackoverflow.com/a/32749533/666727
+*/
+class ExtendableError extends Error {
+  constructor(msg) {
+    super(message);
+    this.name = this.constructor.name;
+    if(typeof Error.captureStackTrace === "function") {
+      Error.captureStackTrace(this, this.constructor);
+    }
+    else {
+      this.stack = (new Error(message)).stack;
+    }
+  }
+}
+
+module.exports = ExtendableError;

lib/FailedTestError.js

@@ -0,0 +1,6 @@
+const ExtendableError = require("./ExtendableError");
+
+class FailedTestError extends ExtendableError {
+}
+
+module.exports = FailedTestError;

lib/FailureDetails.js

@@ -0,0 +1,77 @@
+/**
+ * Stores details about an AVA test failure, which can be used to save and reset variables when re-running tests.
+ * @property {Number} assertCount
+ * @property {Error} [assertError]
+ * @property {Boolean} calledEnd
+ * @property {Number} [duration]
+ * @property {Number} pendingAssertionCount
+ * @property {Number} [planCount]
+ * @property {Number} startedAt
+ * @see https://github.com/avajs/ava/blob/854203a728c4dff3ea61a3bcf49f1dfae04c7657/lib/test.js#L103
+ * @todo Write unit tests
+*/
+class FailureDetails {
+
+  /**
+   * The test properties that need to be saved and reset between test attempts.
+  */
+  static get testProperties() {
+    return [
+      "assertCount",
+      "assertError",
+      "calledEnd",
+      "duration",
+      "pendingAssertionCount",
+      "planCount",
+      "startedAt",
+    ];
+  }
+
+  /**
+   * Copy the test details that should be saved between test attempts.
+   * @param {Test|FailureDetails} source the object to copy details from
+   * @param {Test|FailureDetails} dest the object to store details in
+   * @return {Test|FailureDetails} the destination object
+  */
+  static copyTestDetails(source, dest) {
+    for(const property of this.testProperties) {
+      dest[property] = source[property];
+    }
+    return dest;
+  }
+
+  /**
+   * Extract the internal AVA details after a test failure.
+   * @param {Test} t the current Test object (`t` in AVA)
+   * @return {FailureDetails} details about the test failure.
+   * @todo Create and use a `typedef` for the return type.  Use in other methods.
+  */
+  static getFailureDetails(t) {
+    return this.copyTestDetails(t, new FailureDetails());
+  }
+
+  /**
+   * Restore the internal AVA details after retrying tests.
+   * @param {Test} t the current Test object (`t` in AVA)
+   * @param {FailureDetails} failureDetails the saved details about the test failure
+   * @return {Test} the given Test object.
+  */
+  static restoreFailureDetails(t, failureDetails) {
+    return this.copyTestDetails(failureDetails, t);
+  }
+
+  /**
+   * Reset the internal AVA details to prepare a clean testing environment.
+   * Creates a `{Test}` object from AVA, and copies it's default properties into the given `{Test}`.
+   * @param {Test} t the current Test object (`t` in AVA)
+   * @return {Test} the given Test object.
+  */
+  static resetTest(t) {
+    const AVATest = require("ava/lib/test");
+    const _test = new AVATest();
+    return this.copyTestDetails(_test, t);
+  }
+
+}
+
+module.exports = FailureDetails;

lib/TestDetails.js

@@ -0,0 +1,41 @@
+const merge = require("lodash.merge");
+const FailureDetails = require("./FailureDetails");
+
+/**
+ * Internal data about a single test instance.
+ * @property {Number} index the index of this test in the {@link AVAVerify} suite.
+ * @property {Number} attempt the number of times this test instance has been tried.  `0` for the first run, and `1` for
+ *   the first retry.  Defaults to `0`.
+ * @property {Boolean} shrink user can set this to `false` to disable shrinking.  Shrinking will not be attempted if
+ *   either this is set, or other internal checks fail.  Defaults to `true`.
+ * @property {AVAVerify} [verify] a link back to the {@link AVAVerify} instance.  Not to be used inside AVAVerify, but
+ *   can be used by user tests if needed.
+ * @property {Boolean} failed `true` if the first attempt failed.
+ * @property {FailureDetails} failureDetails details from the first failing run.
+ * @property {Any[]} firstValues the generated values for the first run of this test.
+*/
+class TestDetails {
+
+  constructor(opts) {
+    merge(this, {
+      attempt: 0,
+      shrink: true,
+      failed: false,
+      failureDetails: new FailureDetails(),
+    }, opts);
+    if(typeof this.index !== "number") { throw new TypeError(`'index' is a required field.  Given ${typeof index}.`); }
+  }
+
+  /**
+   * Store the initial generated values for this test.
+   * @param {Any[]} vals the generated values
+   * @return {Any[]} the given values
+  */
+  storeFirstValues(vals) {
+    this.firstValues = vals;
+    return vals;
+  }
+
+}
+
+module.exports = TestDetails;

package.json

@@ -31,6 +31,7 @@
     "jsverify": "^0.8.2"
   },
   "dependencies": {
+    "lodash.merge": "^4.6.0",
     "modconf": "0.0.1"
   },
   "devDependencies": {