chore(conformance): agave 4.0 upgrade

.circleci/config.yml

@@ -68,37 +68,6 @@ jobs:
           name: Run Tests
           command: zig-out/bin/test 2>&1 | cat
 
-  build_linux_release:
-    executor: linux-executor
-    steps:
-      - checkout
-      - attach_workspace:
-          at: workspace
-      - restore_cache:
-          key: linux-x86_64-0.15.2-{{ checksum "build.zig" }}-{{ checksum "build.zig.zon" }}-selfhosted-release
-      - run:
-          name: Build
-          command: |
-            sudo apt-get update -y && sudo apt-get install wget -y
-            ./scripts/proxy_workaround.sh workspace/zig/zig
-            workspace/zig/zig build sig fuzz -Dno-run -Denable-tsan=false -Doptimize=ReleaseSafe -Dcpu=x86_64_v3 -p workspace/zig-out-release --summary all
-      - run:
-          name: Build Conformance
-          command: |
-            cd conformance
-            ../workspace/zig/zig build -Doptimize=ReleaseSafe -Dcpu=x86_64_v3 -p ../workspace/conformance-release --summary all
-      - save_cache:
-          key: linux-x86_64-0.15.2-{{ checksum "build.zig" }}-{{ checksum "build.zig.zon" }}-selfhosted-release
-          paths:
-            - .zig-cache
-            - ~/.cache/zig
-      - persist_to_workspace:
-          root: workspace
-          paths:
-            - "zig-out-release/bin/sig"
-            - "zig-out-release/bin/fuzz"
-            - "conformance-release/lib/libsolfuzz_sig.so"
-
   build_check_macos:
     executor: macos-executor
     steps:
@@ -242,6 +211,19 @@ jobs:
       - checkout
       - attach_workspace:
           at: workspace
+      - restore_cache:
+          key: linux-x86_64-0.15.2-{{ checksum "build.zig" }}-{{ checksum "build.zig.zon" }}-selfhosted-release
+      - run:
+          name: Build
+          command: |
+            sudo apt-get update -y && sudo apt-get install wget -y
+            ./scripts/proxy_workaround.sh workspace/zig/zig
+            workspace/zig/zig build sig fuzz -Dno-run -Denable-tsan=false -Doptimize=ReleaseSafe -Dcpu=x86_64_v3 -p workspace/zig-out-release --summary all
+      - save_cache:
+          key: linux-x86_64-0.15.2-{{ checksum "build.zig" }}-{{ checksum "build.zig.zon" }}-selfhosted-release
+          paths:
+            - .zig-cache
+            - ~/.cache/zig
       - run:
           name: Run Gossip
           command: bash scripts/gossip_test.sh 120 workspace/zig-out-release/bin/sig
@@ -258,6 +240,29 @@ jobs:
           name: Run Ledger Fuzzer
           command: workspace/zig-out-release/bin/fuzz --seed 19 ledger 10000
 
+  build_conformance:
+    executor: linux-executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: workspace
+      - restore_cache:
+          key: build_conformance-v1-{{ checksum "build.zig" }}-{{ checksum "build.zig.zon" }}-{{ checksum "conformance/build.zig" }}-{{ checksum "conformance/build.zig.zon" }}
+      - run:
+          name: Build Conformance
+          command: |
+            cd conformance
+            ../workspace/zig/zig build -p ../workspace/conformance-release --summary all
+      - save_cache:
+          key: build_conformance-v1-{{ checksum "build.zig" }}-{{ checksum "build.zig.zon" }}-{{ checksum "conformance/build.zig" }}-{{ checksum "conformance/build.zig.zon" }}
+          paths:
+            - .zig-cache
+            - ~/.cache/zig
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            - "conformance-release/lib/libsolfuzz_sig.so"
+
   solana_conformance:
     executor: linux-executor
     # We need a pretty high parallelism since the vm_interp harness has a *lot* of fixtures. Perhaps
@@ -267,11 +272,35 @@ jobs:
       - checkout
       - attach_workspace:
           at: workspace
+      - run:
+          name: Prepare /nix permissions so cache restore works
+          command: |
+            sudo mkdir -p /nix
+            sudo chown "$USER" /nix
       - restore_cache:
-          key: linux-x86_64-0.15.2-{{ checksum "build.zig" }}-{{ checksum "build.zig.zon" }}-selfhosted-release
+          key: nix-v1-{{ checksum "conformance/flake.nix" }}-{{ checksum "conformance/flake.lock" }}-{{ checksum "conformance/commits.env" }}
+      - run:
+          name: Install nix
+          command: |
+            sh <(curl --proto '=https' --tlsv1.2 -L https://nixos.org/nix/install) --no-daemon
+            mkdir -p ~/.config/nix
+            echo 'experimental-features = nix-command flakes' > ~/.config/nix/nix.conf
+            echo '. /home/circleci/.nix-profile/etc/profile.d/nix.sh' >> "$BASH_ENV"
       - run:
           name: Run Fixtures
-          command: ./conformance/scripts/ci-run.sh
+          command: |
+            cd conformance
+            nix develop \
+              -s SPLIT_TESTS true \
+              -s PREBUILT_LIB_DIR ../workspace/conformance-release/lib \
+              -c ./scripts/ci-run.sh
+      - save_cache:
+          key: nix-v1-{{ checksum "conformance/flake.nix" }}-{{ checksum "conformance/flake.lock" }}-{{ checksum "conformance/commits.env" }}
+          paths:
+            - /nix/store
+            - /nix/var/nix/db
+            - /nix/var/nix/profiles
+            - ~/.cache/nix
 
   linux_misc_checks:
     executor: linux-executor
@@ -301,9 +330,6 @@ jobs:
       - run:
           name: Check docs
           command: python3 docs/check.py ./
-      - run:
-          name: Check conformance commits
-          command: conformance/scripts/verify-commits.sh
       - run:
           name: V2 checks
           command: |
@@ -333,7 +359,7 @@ workflows:
       - test_kcov_linux:
           requires:
             - setup_zig_linux
-      - build_linux_release:
+      - gossip_and_fuzz:
           requires:
             - setup_zig_linux
       - build_and_test_linux:
@@ -342,12 +368,12 @@ workflows:
       - linux_misc_checks:
           requires:
             - setup_zig_linux
-      - solana_conformance:
+      - build_conformance:
           requires:
-            - build_linux_release
-      - gossip_and_fuzz:
+            - setup_zig_linux
+      - solana_conformance:
           requires:
-            - build_linux_release
+            - build_conformance
 
   check_macos:
     jobs:

conformance/README.md

@@ -1,24 +1,83 @@
-Dependencies:
-- python 3.11
-- cargo/rust
-- zig 0.15.2
-- git 2.49
-- cmake
+# Dependencies
+
+Install [nix](https://nixos.org/download/) if you do not have it.
+
+```bash
+nix develop            # if you only want to run the tests
+nix develop .#agave    # if you also want to run the test vectors against agave
+```
+
+# Build
+
+To run the conformance tests, you'll need a build of solfuzz_sig. Either Debug or ReleaseSafe builds are fine.
 
 ```bash
-# set up the test environment
-scripts/setup-env.sh
-source env/pyvenv/bin/activate
+zig build solfuzz_sig
+```
 
-# compile the sig binary to test
-zig build -Doptimize=ReleaseSafe solfuzz_sig
 
-# run the conformance tests for the first time
-./run.py --create
+# Run
 
-# re-run conformance tests without re-creating the fixtures from scratch
-./run.py
+**Run the test vectors**
 
-# for more options
-./run.py --help
+```bash
+# run all conformance test vectors
+run
+
+# run only the transaction harness tests
+run txn
+```
+
+For customization, try `--help`.
+
+**Run the conformance CI job**
+
+```bash
+scripts/ci-run.sh
+```
+
+**Run solana-conformance directly**
+
+run.py is a helper script to make this process easier. But in some cases when debugging, it's useful to interact directly with solana-conformance. Here's an example of how to run the zk_sdk tests with solana-conformance:
+
+```bash
+solana-conformance \
+    exec-fixtures \
+    -i env/test-vectors/instr/fixtures/zk_sdk \
+    -t zig-out/lib/libsolfuzz_sig.so \
+    -o env/test-outputs/
 ```
+
+**Debug Agave**
+
+If you want to debug agave or manually generate fixtures from agave, you'll need to set up the agave environment with `nix develop .#agave`.
+
+You can edit any of the agave code in the env/ folder to debug agave. To run the conformance tests against this code, you'll need to compile solfuzz_agave:
+
+```bash
+cd env/solfuzz-agave
+cargo build --lib --release
+```
+
+Then you can use some commands like these:
+
+```bash
+# run the tests against agave
+run --exec-lib env/solfuzz-agave/target/release/libsolfuzz_agave.so
+
+# create the fixtures based on agave, and run the conformance tests against sig
+run --create
+
+# re-run conformance tests against sig using your created fixtures
+run --use-created
+```
+
+# Interpreting results
+
+The expected and actual results for all tests are in env/test-outputs. You can use `parseout` to interpret the results.
+
+```bash
+parseout txn
+```
+
+See [parseout/README.md] for more info.

conformance/build.zig

@@ -15,7 +15,7 @@ pub fn build(b: *Build) void {
     const solfuzz_sig_step = b.step("solfuzz_sig", "The solfuzz sig library.");
     const test_step = b.step("test", "Run unit tests");
 
-    // current commit: 3dea3808eab107f3010ddd3bbc24a31cc4b07c40
+    // current commit in commits.env
     const proto_step = b.step(
         "protobuf",
         "Re-generate protobuf definitions based on the `protosol` directory." ++
@@ -91,7 +91,6 @@ pub fn build(b: *Build) void {
     const protoc_run = pb.RunProtocStep.create(pb_dep.builder, target, .{
         .destination_directory = b.path("src/proto"),
         .source_files = &.{
-            "protosol/proto/elf.proto",
             "protosol/proto/vm.proto",
             "protosol/proto/shred.proto",
             "protosol/proto/txn.proto",

conformance/commits.env

@@ -1,11 +1,16 @@
 # The commits currently used in CI for sig's conformance with agave.
 
-SOLANA_CONFORMANCE_COMMIT=70b5ee3a73c3e12eb0c2f2bfd3037e5645903c94
-TEST_VECTORS_COMMIT=0a97cc29937eaa6b31e27448f2c43adf758f021c
-SOLFUZZ_AGAVE_COMMIT=3e12a4b269083d16b918bc8885c89fa6f6b556d7
+SOLANA_CONFORMANCE_COMMIT="786798b9536f310cd6fe8ba8a05df0a187500cd2"
+TEST_VECTORS_COMMIT="d6b81e3de1132f5032aabcd8a975e306634ae13d"
+SOLFUZZ_AGAVE_COMMIT="51c1f5c8bc38748c57aa9fb4b51ce9501111c010"
 
 # protosol definitions needed for the above commit of solfuzz-agave
-AGAVE_PROTOSOL_COMMIT=3dea3808eab107f3010ddd3bbc24a31cc4b07c40
+AGAVE_PROTOSOL_COMMIT="e52ca0f21f4ef68e8f2ed5269dcfaa8b50091f7f"
 
 # protosol definitions used for sig
-SIG_PROTOSOL_COMMIT=3dea3808eab107f3010ddd3bbc24a31cc4b07c40
+SIG_PROTOSOL_COMMIT="e52ca0f21f4ef68e8f2ed5269dcfaa8b50091f7f"
+
+# these are the commits used by solfuzz_agave.
+# they are not needed directly our ci, but are useful for local debugging
+AGAVE_COMMIT="8d35051b7cac869bd165ae440db672617d053026"
+SBPF_COMMIT="d32556248756e19ee49a44a94ccb633e2414a386"