diff --git a/docs/design/dfx-testing-story-example/dfx.json b/docs/design/dfx-testing-story-example/dfx.json new file mode 100644 index 0000000000..3eb81cbc4d --- /dev/null +++ b/docs/design/dfx-testing-story-example/dfx.json @@ -0,0 +1,31 @@ +{ + "canisters": { + "cleanSheets": { + "_comment:": "we list `main` service code versus its `unit-tests` (each is a single Motoko function to run) versus its `functional-tests` (each is an `install-and-go`), versus `e2e-tests` that each involve more than a single `install-and-go`, and require a `test-script` over several canisters." + + "main": "src/cleanSheets.mo" + "unit-tests": { + "all": "test/main.mo" + }, + "func-tests": { + "functionalTest1": "test/simpleAdaptonDivByZero.mo", + "functionalTest2": "test/simpleExcelEmulation.mo", + }, + "e2e-tests": { + "AliceBob": { + "_comment:": "test-script installs and scripts bots `alice` and `bob`" + "test-script": "test/aliceBob/test-script" + "aliceCanister": "test/aliceBob/aliceCanister.mo" + "bobCanister": "test/aliceBob/bobCanister.mo" + } + "AliceBobCharlie": { + "_comment:": "reuse bots `alice` and `bob`, and includes bot `charlie`" + "test-script": "test/aliceBobCharlie/test-script" + "aliceCanister": "test/aliceBob/aliceCanister.mo" + "bobCanister": "test/aliceBob/bobCanister.mo" + "charlieCanister": "test/aliceBobCharlie/charlieCanister.mo" + } + } + }, + } +} diff --git a/docs/design/dfx-testing-story-example/src/adapton.mo b/docs/design/dfx-testing-story-example/src/adapton.mo new file mode 100644 index 0000000000..764f9f5596 --- /dev/null +++ b/docs/design/dfx-testing-story-example/src/adapton.mo @@ -0,0 +1 @@ +(OMITTING EXAMPLE CODE) diff --git a/docs/design/dfx-testing-story-example/src/cleanSheets.mo b/docs/design/dfx-testing-story-example/src/cleanSheets.mo new file mode 100644 index 0000000000..764f9f5596 --- /dev/null +++ b/docs/design/dfx-testing-story-example/src/cleanSheets.mo @@ -0,0 +1 @@ +(OMITTING EXAMPLE CODE) diff --git a/docs/design/dfx-testing-story-example/src/eval.mo b/docs/design/dfx-testing-story-example/src/eval.mo new file mode 100644 index 0000000000..764f9f5596 --- /dev/null +++ b/docs/design/dfx-testing-story-example/src/eval.mo @@ -0,0 +1 @@ +(OMITTING EXAMPLE CODE) diff --git a/docs/design/dfx-testing-story-example/src/main.mo b/docs/design/dfx-testing-story-example/src/main.mo new file mode 100644 index 0000000000..5ae6c68efa --- /dev/null +++ b/docs/design/dfx-testing-story-example/src/main.mo @@ -0,0 +1 @@ +(EXAMPLE CODE OMITTED) diff --git a/docs/design/dfx-testing-story-example/src/types.mo b/docs/design/dfx-testing-story-example/src/types.mo new file mode 100644 index 0000000000..764f9f5596 --- /dev/null +++ b/docs/design/dfx-testing-story-example/src/types.mo @@ -0,0 +1 @@ +(OMITTING EXAMPLE CODE) diff --git a/docs/design/dfx-testing-story-example/test/aliceBob/aliceCanister.mo b/docs/design/dfx-testing-story-example/test/aliceBob/aliceCanister.mo new file mode 100644 index 0000000000..5ae6c68efa --- /dev/null +++ b/docs/design/dfx-testing-story-example/test/aliceBob/aliceCanister.mo @@ -0,0 +1 @@ +(EXAMPLE CODE OMITTED) diff --git a/docs/design/dfx-testing-story-example/test/aliceBob/bobCanister.mo b/docs/design/dfx-testing-story-example/test/aliceBob/bobCanister.mo new file mode 100644 index 0000000000..5ae6c68efa --- /dev/null +++ b/docs/design/dfx-testing-story-example/test/aliceBob/bobCanister.mo @@ -0,0 +1 @@ +(EXAMPLE CODE OMITTED) diff --git a/docs/design/dfx-testing-story-example/test/aliceBob/test-script b/docs/design/dfx-testing-story-example/test/aliceBob/test-script new file mode 100644 index 0000000000..feb88b1290 --- /dev/null +++ b/docs/design/dfx-testing-story-example/test/aliceBob/test-script @@ -0,0 +1,10 @@ +install bobCanister +==> #ok(bobId) +install aliceCanister +==> #ok(aliceId) +call bobId addFriend aliceId +==> #ok(()) +call aliceId addFriend BobId +==> #ok(()) +call bobId collaborateWithAlice +==> #ok(()) \ No newline at end of file diff --git a/docs/design/dfx-testing-story-example/test/aliceBobCharlie/charlieCanister.mo b/docs/design/dfx-testing-story-example/test/aliceBobCharlie/charlieCanister.mo new file mode 100644 index 0000000000..5ae6c68efa --- /dev/null +++ b/docs/design/dfx-testing-story-example/test/aliceBobCharlie/charlieCanister.mo @@ -0,0 +1 @@ +(EXAMPLE CODE OMITTED) diff --git a/docs/design/dfx-testing-story-example/test/aliceBobCharlie/test-script b/docs/design/dfx-testing-story-example/test/aliceBobCharlie/test-script new file mode 100644 index 0000000000..c30a278352 --- /dev/null +++ b/docs/design/dfx-testing-story-example/test/aliceBobCharlie/test-script @@ -0,0 +1,11 @@ +install bobCanister ==> #ok(bobId) +install aliceCanister ==> #ok(aliceId) +install charlieCanister ==> #ok(charlieId) +# comment out these calls, to try alternatives where Charlie starts all interactions +# call bobId addFriend aliceId ==> #ok(()) +# call bobId addFriend charlieId ==> #ok(()) +# call aliceId addFriend bobId ==> #ok(()) +# call aliceId addFriend charlieId ==> #ok(()) +call charlieId addFriend bobId ==> #ok(()) +call charlieId addFriend aliceId ==> #ok(()) +call charlieId collaborateWithAliceAndBob ==> #ok(()) diff --git a/docs/design/dfx-testing-story-example/test/simpleAdaptonDivByZero.mo b/docs/design/dfx-testing-story-example/test/simpleAdaptonDivByZero.mo new file mode 100644 index 0000000000..0833023a42 --- /dev/null +++ b/docs/design/dfx-testing-story-example/test/simpleAdaptonDivByZero.mo @@ -0,0 +1,208 @@ +import R "mo:stdlib/result"; +import P "mo:stdlib/prelude"; + +import T "../src/types"; +import A "../src/adapton"; +import E "../src/eval"; + +/* + +This file reproduces the intro example from the Adapton Rust docs, +found here: + + https://docs.rs/adapton/0/adapton/#demand-driven-change-propagation + +The DCG update behavior asserted here (see uses of +`assertLogEventLast` below) matches the cleaning/dirtying behavior +described in the link above, and in other documents and papers. + +In particular, the same example is used in a recorded adapton talk, +from the http://adapton.org website. The link above has slides with +still pictures from the talk. + +*/ + +actor SimpleAdaptonDivByZero { + + public func go() { + let ctx : T.Adapton.Context = A.init(true); + + // "cell 1 holds 42": + let cell1 : T.Adapton.NodeId = assertOkPut(A.put(ctx, #nat(1), #nat(42))); + A.assertLogEventLast + (ctx, #put(#nat(1), #nat(42), [])); + + // "cell 2 holds 2": + let cell2 : T.Adapton.NodeId = assertOkPut(A.put(ctx, #nat(2), #nat(2))); + A.assertLogEventLast + (ctx, #put(#nat(2), #nat(2), [])); + + // "cell 3 holds a suspended closure for this expression: + // + // get(cell1) / get(cell2) + // + // ...and it is still unevaluated". + // + let cell3 : T.Adapton.NodeId = assertOkPut( + A.putThunk(ctx, #nat(3), + E.closure( + null, + #strictBinOp(#div, + #get(#refNode(cell1)), + #get(#refNode(cell2)) + ))) + ); + + // "cell 4 holds a suspended closure for this expression: + // + // if (get(cell2) == 0) { 0 } + // else { get(cell3) } + // + // ...and it is still unevaluated". + // + let cell4 : T.Adapton.NodeId = assertOkPut( + A.putThunk(ctx, #nat(4), + E.closure( + null, + #ifCond(#strictBinOp(#eq, + #get(#refNode(cell2)), + #nat(0)), + #nat(0), + #get(#refNode(cell3))))) + ); + + // demand division: + let res1 = assertOkGet(A.get(ctx, cell4)); + A.assertLogEventLast(ctx, + #get(#nat(4), #ok(#nat(21)), + [ + #evalThunk( + #nat(4), #ok(#nat(21)), + [#get(#nat(2), #ok(#nat(2)), []), + #get(#nat(3), #ok(#nat(21)), + [ + #evalThunk( + #nat(3), #ok(#nat(21)), + [ + #get(#nat(1), #ok(#nat(42)), []), + #get(#nat(2), #ok(#nat(2)), []) + ]) + ]) + ]) + ]) + ); + + // "cell 2 holds 0": + ignore A.put(ctx, #nat(2), #nat(0)); + A.assertLogEventLast + (ctx, + #put(#nat(2), #nat(0), + [ + #dirtyIncomingTo( + #nat(2), + [ + #dirtyEdgeFrom( + #nat(3), + [ + #dirtyIncomingTo( + #nat(3), + [ + #dirtyEdgeFrom( + #nat(4), + [ + #dirtyIncomingTo(#nat(4), []) + ]) + ]) + ]), + #dirtyEdgeFrom( + #nat(4), + [ + #dirtyIncomingTo( + #nat(4), []) + ]) + ]) + ]) + ); + + // re-demand division: + let res2 = assertOkGet(A.get(ctx, cell4)); + A.assertLogEventLast + (ctx, + #get(#nat(4), #ok(#nat(0)), + [ + #cleanThunk( + #nat(4), false, + [ + #cleanEdgeTo( + #nat(2), false, []) + ]), + #evalThunk( + #nat(4), #ok(#nat(0)), + [ + #get(#nat(2), #ok(#nat(0)), []) + ]) + ])); + + // "cell 2 holds 2": + ignore A.put(ctx, #nat(2), #nat(2)); + A.assertLogEventLast + (ctx, + #put(#nat(2), #nat(2), + [ + #dirtyIncomingTo( + #nat(2), + [ + #dirtyEdgeFrom( + #nat(4), + [ + #dirtyIncomingTo( + #nat(4), []) + ]) + ]) + ])); + + // re-demand division: + let res3 = assertOkGet(A.get(ctx, cell4)); + A.assertLogEventLast + (ctx, + #get(#nat(4), #ok(#nat(21)), + [ + #cleanThunk( + #nat(4), false, + [ + #cleanEdgeTo(#nat(2), false, []) + ]), + #evalThunk( + #nat(4), #ok(#nat(21)), + [ + #get(#nat(2), #ok(#nat(2)), []), + #get(#nat(3), #ok(#nat(21)), + [ + #cleanThunk( + #nat(3), true, + [ + #cleanEdgeTo(#nat(1), true, []), + #cleanEdgeTo(#nat(2), true, []) + ]) + ]) + ]) + ])); + }; + + + func assertOkPut(r:R.Result) : T.Adapton.NodeId { + switch r { + case (#ok(id)) { id }; + case _ { P.unreachable() }; + } + }; + + func assertOkGet(r:R.Result) : T.Eval.Result { + switch r { + case (#ok(res)) { res }; + case _ { P.unreachable() }; + } + }; +}; + +//SimpleAdaptonDivByZero.go(); diff --git a/docs/design/dfx-testing-story-example/test/simpleExcelEmulation.mo b/docs/design/dfx-testing-story-example/test/simpleExcelEmulation.mo new file mode 100644 index 0000000000..e23030835e --- /dev/null +++ b/docs/design/dfx-testing-story-example/test/simpleExcelEmulation.mo @@ -0,0 +1,77 @@ +import R "mo:stdlib/result"; +import P "mo:stdlib/prelude"; +import Debug "mo:stdlib/debug"; + +import T "../src/types"; +import A "../src/adapton"; +import E "../src/eval"; + +actor simpleExcelEmulation { + + public func go() { + + let sheetExp : T.Eval.Exp = + #sheet( + #text("S"), + [ + [ #nat(1), #nat(2) ], + [ #strictBinOp(#add, + #cellOcc(0,0), + #cellOcc(0,1)), + #strictBinOp(#mul, + #nat(2), + #cellOcc(1,0)) ] + ]); + + // Adapton maintains our dependence graph + let actx : T.Adapton.Context = A.init(true); + + // create the initial Sheet datatype from the DSL expression above + let s : T.Sheet.Sheet = { + switch (E.evalExp(actx, null, sheetExp)) { + case (#ok(#sheet(s))) s; + case _ { P.unreachable() }; + }}; + + // Demand that the sheet's results are fully refreshed + ignore E.Sheet.refresh(actx, s); + ignore E.Sheet.refresh(actx, s); + A.assertLogEventLast( + actx, + #get(#tagTup(#text("S"), [#nat(1), #nat(1), #text("out")]), #ok(#nat(6)), []) + ); + + // Update the sheet by overwriting (0,0) with a new formula: + ignore E.Sheet.update(actx, s, 0, 0, + #strictBinOp(#add, #nat(666), #cellOcc(0,1))); + + // Demand that the sheet's results are fully refreshed + ignore E.Sheet.refresh(actx, s); + ignore E.Sheet.refresh(actx, s); + assert (s.errors.len() == 0); + A.assertLogEventLast( + actx, + #get(#tagTup(#text("S"), [#nat(1), #nat(1), #text("out")]), #ok(#nat(1_340)), []) + ); + + // Update the sheet, creating a cycle at (0,0): + ignore E.Sheet.update(actx, s, 0, 0, #cellOcc(0,0)); + ignore E.Sheet.refresh(actx, s); + assert (s.errors.len() != 0); + + // Update the sheet, removing the cycle: + ignore E.Sheet.update(actx, s, 0, 0, + #strictBinOp(#add, #nat(666), #cellOcc(0,1))); + + // Demand that the sheet's results are fully refreshed + ignore E.Sheet.refresh(actx, s); + ignore E.Sheet.refresh(actx, s); + assert (s.errors.len() == 0); + A.assertLogEventLast( + actx, + #get(#tagTup(#text("S"), [#nat(1), #nat(1), #text("out")]), #ok(#nat(1_340)), []) + ); + }; +}; + +//simpleExcelEmulation.go() diff --git a/docs/design/dfx-testing-story.adoc b/docs/design/dfx-testing-story.adoc new file mode 100644 index 0000000000..263c675a2f --- /dev/null +++ b/docs/design/dfx-testing-story.adoc @@ -0,0 +1,247 @@ +## Design Doc: DFX Testing Story + +Creator(s): Matthew Hammer, after discussion with and input from Hans Larsen, on 2020-02-03. + +### Terms and goals + + - unit tests --- one canister for all the unit tests, together; + a standard library provides the framework for this canister (TBD). + + - functional tests --- one canister for each test, built and run separately. + + - e2e tests --- many canisters each; each requires a `test-script` to coordinate. + + +In CI, we want to write, as humans and as CI scripts: + +`dfx test` + +And variations that permit more control over what tests run: + + - `dfx test unit` for "Run all unit tests for all libraries used by this project" (we + currently recompile these transitive libraries ourselves, so this + "transitive testing" is a reasonable thing to do, too). + + - `dfx test func` for "Run all functional tests". + + - `dfx test func X` for "Run only the functional test named X". + + - `dfx test e2e` for "Run all end-to-end tests for the project". + + - `dfx test e2e X` for "Run only end-to-end test named X". + +We consider e2e tests and unit tests in the parts below. + +These classes of tests are quite different in their requirements of +the `dfx` tool. The functional tests provide a common half-way point; +see the example project tree for more details. + + +### Part 1: Authoring a CI-driven, end-to-end (e2e) test of a Motoko project + +Usecase: _We want to author a Motoko project, and have CI rerun a +suite of end-to-end (e2e) tests for each commit, e.g., using Travis +CI, or something similar._ + +#### Simplifying assumptions + +- we do not discuss building canisters here; we assume they have been + built somehow; this design is about testing, not building. + +- each e2e test consists of starting a replica, running one or more + "test actions" (see below), and then stopping the replica. + +- two distinct e2e tests are independent; they do not share any + service state; they only share overlapping source code for canisters + (probably). + +- distinct e2e tests can be run in sequence, in parallel, or even on + distinct replicas (if necessary; not a priority yet). + +In the remainder of this document, we focus on the case of a single +e2e test in isolation, knowing that each _additional_ e2e test is +always an independent (additional/separable) concern. + +Each e2e test is given by a single sequence of test actions. We refer +to this sequence as the test's _action sequence_. We use the term +"test script" and "action sequence" interchangeably throughout. + +#### Test actions + +Each "test action" is a scriptable dfx command with an associated +response that it expects, perhaps as a _pattern_. + +` ::= ` (three distinct cases:) + +- `| dfx-canister-install ` +- `| dfx-canister-call ` +- `| dfx-canister-query ` + +In each case, the last part of the action gives a "dfx result +pattern", which permits the sequence to destructure results and bind +variables, which may appear as either the `` or within the +`` of subsequent actions of the same sequence. + +**Static checks:** The test system statically checks a test's action +sequence before running it; consequently, variables cannot be used in +the sequence without being first defined earlier in the sequence. We +do not check the IDL result and argument types match, however (or do +we want to do that too?). + +#### Dfx result patterns + +Each time the test script gets a response from an action (`install` or +`call` or `query`) we may either want to bind that value to a test +script variable (to use in a subsequent action), or insist that that +value is a certain value, or both. + +The dfx result pattern grammar permits each of these uses: + +``` + ::= bind + | match + + ::= + | (, ..., ) + | record { lab_1: , ..., lab_k: } + | variant { lab } + | + | + | + | ( as ) +...TODO +``` + +The second case (`match`) is a generalization of the first one +(`bind`), as it binds variables as one possible usecase. + +More generally, it consists of patterns that each attempt to match an +IDL value, destructure it, and bind its parts to zero or more variable +names. The grammar above should agree with the IDL value syntax (but +probably doesn't yet). + +In either the `bind` or `match` cases, the test variables being +defined by the action's result exist within the scope of the remaining +actions in the same sequence, but not outside of it. These variables +are useful for forming functional chains of test calls within the +script symbolically, where they avoid the script explicitly mentioning +all of the intermittent input and output values explicitly, as IDL +values. + +#### Technical clarification on error detection and reporting: + +The result pattern-matching logic alluded to above (for the + grammar) should build on the existing Rust +codebase for Candid, not be based on commands within bash, over +stringified versions, etc. In particular, the test fails with special +kinds of test errors if this scripting syntax is malformed in some +way; likewise, the test actions themselves must follow a formal +syntax, and they emit associated errors, if at all, before the test +suite runs any test actions. + +#### Remaining details: + +Many practical details remain to nail down, but the sketch above gives +the most important definitions; we still need a discipline for listing +multiple e2e tests within the `dfx.json` file, perhaps with distinct +names, so that `dfx test e2e foo` and `dfx test e2e bar` do different +things. + +Likewise, we may want regex logic over these names, so that `dfx test +e2e foo*` and `dfx test e2e bar*Baz` mean and do the expected things. +This is all TBD. + + +### Part 2: Unit tests on canister Wasm from Motoko source + +Suppose a developer writes a Motoko library `FooLib` with a set of +public functions and a set of (publicly visible) unit test functions +that exercise the other (public or private) functions of `FooLib`. + +Currently, we have a story for a Motoko project to import `FooLib` and +use its public interface when it can point to `FooLib` somewhere on the +local filesystem. + +**However, we have no story for the client program running the unit +tests of FooLib. Where does it look to find them?** + +When unit tests (or functional tests) only exercise the _public_ +interface of FooLib, and _never enter its private implementation +directly_, we may want these test functions to exist outside of the +implementation file/directory for FooLib. Currently, we organize +the Motoko `stdlib` source and test trees as parallel, but distinct: + +(Omitting all but two modules, for concision:) +``` +/src/buf.mo +/src/hash.mo +/test/bufTest.mo +/test/hashTest.mo +``` + +#### Key Assumption, to simplify public/private visibility: + +Let's assume that all Motoko code that we want to directly unit test +is public in its defining module. + +Under this assumption, we can assume that testing code is separable +from implementation code, and the `src` versus `test` directory +distinction is workable. + +Of course, this implementation module versus test module separation +does not make sense for unit tests of private functions. Those tests +require a different approach, where the (publicly-exposed) test code +is mixed into the module itself, where it can access private members +to test them. While this usecase is important, it's a complex use +case in terms of visibility (public test and private code being +tested), so let's set it aside for now. + +#### Remaining details: + +- How does `dfx` discover the test functions of the `test` modules? +- For instance, can it discover all of the public methods of each test module and merely assume each is a unit test? +- Other details that are similar to those of Part 1 with respect to `dfx.json`. + + + +### Example project files + +To help nail down the remaining details listed above, this proposal +includes an example project tree, illustrating how the `dfx.json` file +relates to this tree, and the files that it contains, including the +canister source that is just for tests, and the test scripts that +install and call these canisters: + +``` +dfx-testing-story-example +├── dfx.json +├── src +│   ├── adapton.mo +│   ├── cleanSheets.mo +│   ├── eval.mo +│   ├── main.mo +│   └── types.mo +└── test + ├── aliceBob + │   ├── aliceCanister.mo + │   ├── bobCanister.mo + │   └── test-script + ├── aliceBobCharlie + │   ├── charlieCanister.mo + │   └── test-script + ├── simpleAdaptonDivByZero.mo + └── simpleExcelEmulation.mo +``` + +The tree contains two example test scripts, for a 2-party (2-canister) +interaction, and for a 3-party (3-canister) interaction. + +The example test scripts (the `test-script` files) in this project +tree give example-based proposals for concrete syntax for the test +scripts, and their test actions. + +Note: The Motoko-like pattern syntax in those files should be replaced +with syntax that is consistent with Candid's concrete syntax +definitions. (the Motoko-like syntax is generally shorter and +more readable to me.) +