Problem: Integrate testground to run benchmark on cluster

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Improvements
 
 * [#1613](https://github.com/crypto-org-chain/cronos/pull/1613) Check admin sender for MsgStoreBlockList in check tx.
+* [#1431](https://github.com/crypto-org-chain/cronos/pull/1431) Integrate testground to run benchmark on cluster.
 
 *Sep 24, 2024*
 

cmd/cronosd/cmd/root.go

@@ -125,6 +125,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig ethermint.EncodingConfig
 		WrapGenTxCmd(encodingConfig.TxConfig, banktypes.GenesisBalancesIterator{}, app.DefaultNodeHome),
 		WrapValidateGenesisCmd(),
 		AddGenesisAccountCmd(app.DefaultNodeHome),
+		genutilcli.AddBulkGenesisAccountCmd(app.DefaultNodeHome),
 		tmcli.NewCompletionCmd(rootCmd, true),
 		ethermintclient.NewTestnetCmd(app.ModuleBasics, banktypes.GenesisBalancesIterator{}),
 		debug.Cmd(),

flake.nix

@@ -31,22 +31,21 @@
         let
           pkgs = import nixpkgs {
             inherit system;
-            overlays = [
-              (import ./nix/build_overlay.nix)
-              poetry2nix.overlays.default
-              gomod2nix.overlays.default
-              self.overlay
-            ];
+            overlays = self.overlays.default;
             config = { };
           };
         in
         rec {
           packages = pkgs.cronos-matrix // {
-            inherit (pkgs) rocksdb;
+            inherit (pkgs) rocksdb testground-image;
           };
           apps = {
             cronosd = mkApp packages.cronosd;
             cronosd-testnet = mkApp packages.cronosd-testnet;
+            stateless-testcase = {
+              type = "app";
+              program = "${pkgs.benchmark-testcase}/bin/stateless-testcase";
+            };
           };
           defaultPackage = packages.cronosd;
           defaultApp = apps.cronosd;
@@ -73,62 +72,20 @@
         }
       )
     ) // {
-      overlay = final: super: {
-        go = super.go_1_22;
-        test-env = final.callPackage ./nix/testenv.nix { };
-        bundle-exe = final.pkgsBuildBuild.callPackage nix-bundle-exe { };
-        # make-tarball don't follow symbolic links to avoid duplicate file, the bundle should have no external references.
-        # reset the ownership and permissions to make the extract result more normal.
-        make-tarball = drv: final.runCommand "tarball-${drv.name}"
-          {
-            nativeBuildInputs = with final.buildPackages; [ gnutar gzip ];
-          } ''
-          tar cfv - -C "${drv}" \
-            --owner=0 --group=0 --mode=u+rw,uga+r --hard-dereference . \
-            | gzip -9 > $out
-        '';
-        bundle-win-exe = drv: final.callPackage ./nix/bundle-win-exe.nix { cronosd = drv; };
-      } // (with final;
-        let
-          matrix = lib.cartesianProductOfSets {
-            network = [ "mainnet" "testnet" ];
-            pkgtype = [
-              "nix" # normal nix package
-              "bundle" # relocatable bundled package
-              "tarball" # tarball of the bundle, for distribution and checksum
-            ];
+      overlays.default = [
+        (import ./nix/build_overlay.nix)
+        poetry2nix.overlays.default
+        gomod2nix.overlays.default
+        (import ./testground/benchmark/overlay.nix)
+        (final: super: {
+          go = super.go_1_22;
+          test-env = final.callPackage ./nix/testenv.nix { };
+          cronos-matrix = final.callPackage ./nix/cronos-matrix.nix {
+            inherit rev;
+            bundle-exe = final.pkgsBuildBuild.callPackage nix-bundle-exe { };
           };
-          binaries = builtins.listToAttrs (builtins.map
-            ({ network, pkgtype }: {
-              name = builtins.concatStringsSep "-" (
-                [ "cronosd" ] ++
-                lib.optional (network != "mainnet") network ++
-                lib.optional (pkgtype != "nix") pkgtype
-              );
-              value =
-                let
-                  cronosd = callPackage ./. {
-                    inherit rev network;
-                  };
-                  bundle =
-                    if stdenv.hostPlatform.isWindows then
-                      bundle-win-exe cronosd
-                    else
-                      bundle-exe cronosd;
-                in
-                if pkgtype == "bundle" then
-                  bundle
-                else if pkgtype == "tarball" then
-                  make-tarball bundle
-                else
-                  cronosd;
-            })
-            matrix
-          );
-        in
-        {
-          cronos-matrix = binaries;
-        }
-      );
+          testground-image = final.callPackage ./nix/testground-image.nix { };
+        })
+      ];
     };
 }

go.mod

@@ -241,7 +241,7 @@ replace (
 	github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0
 	github.com/cometbft/cometbft-db => github.com/crypto-org-chain/cometbft-db v0.0.0-20231011055109-57922ac52a63
 	// release/v0.47.x
-	github.com/cosmos/cosmos-sdk => github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20240517084251-5ba20d99ec23
+	github.com/cosmos/cosmos-sdk => github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20241008100047-e671aed24d19
 	github.com/crypto-org-chain/cronos/memiavl => ./memiavl
 	github.com/crypto-org-chain/cronos/store => ./store
 	github.com/crypto-org-chain/cronos/versiondb => ./versiondb

go.sum

@@ -489,8 +489,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/crypto-org-chain/cometbft-db v0.0.0-20231011055109-57922ac52a63 h1:R1QJ9a3XdYMSKo+1RdFifxb/g3lNypC52L/rpYrWoKo=
 github.com/crypto-org-chain/cometbft-db v0.0.0-20231011055109-57922ac52a63/go.mod h1:rocwIfnS+kA060x64gkSIRvWB9StSppIkJuo5MWzL24=
-github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20240517084251-5ba20d99ec23 h1:pmpedXv3cd18uDwv62mfevmEVa0KCMgR/hlsTRU9p8g=
-github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20240517084251-5ba20d99ec23/go.mod h1:vdMQLvLSGh56GqAAQP/w2R389j+g+hHOIwKcUNt9yAc=
+github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20241008100047-e671aed24d19 h1:e45niobYERw7QF6viZe5KhIH8eLgDTtg+mT5HsVhqCs=
+github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20241008100047-e671aed24d19/go.mod h1:vdMQLvLSGh56GqAAQP/w2R389j+g+hHOIwKcUNt9yAc=
 github.com/crypto-org-chain/ethermint v0.6.1-0.20240924033445-e55b15cf1494 h1:PsBCicxLYx9u8LXgqoL1DYYQXACank9z1FNkwBqz/tY=
 github.com/crypto-org-chain/ethermint v0.6.1-0.20240924033445-e55b15cf1494/go.mod h1:zJYmx1D+tDggzvXmdJ0/I62TeIykdCMfJBjBxOcniAU=
 github.com/crypto-org-chain/go-ethereum v1.10.20-0.20231207063621-43cf32d91c3e h1:vnyepPQ/m25+19xcTuBUdRxmltZ/EjVWNqEjhg7Ummk=

gomod2nix.toml

@@ -142,8 +142,8 @@ schema = 3
     version = "v1.0.0-beta.5"
     hash = "sha256-Fy/PbsOsd6iq0Njy3DVWK6HqWsogI+MkE8QslHGWyVg="
   [mod."github.com/cosmos/cosmos-sdk"]
-    version = "v0.46.0-beta2.0.20240517084251-5ba20d99ec23"
-    hash = "sha256-rmW8n5Vs2E7AVKVGiUy344uGtL8+LBDqZHOHQVF44YA="
+    version = "v0.46.0-beta2.0.20241008100047-e671aed24d19"
+    hash = "sha256-6JliyJyA6rm80kPKz3tFtfkFRPEzzQgd+otKzMCeAPI="
     replaced = "github.com/crypto-org-chain/cosmos-sdk"
   [mod."github.com/cosmos/go-bip39"]
     version = "v1.0.0"

nix/cronos-matrix.nix

@@ -0,0 +1,56 @@
+{ lib
+, stdenv
+, callPackage
+, buildPackages
+, runCommand
+, bundle-exe
+, rev ? "dirty"
+}:
+let
+  # make-tarball don't follow symbolic links to avoid duplicate file, the bundle should have no external references.
+  # reset the ownership and permissions to make the extract result more normal.
+  make-tarball = drv: runCommand "tarball-${drv.name}"
+    {
+      nativeBuildInputs = with buildPackages; [ gnutar gzip ];
+    } ''
+    tar cfv - -C "${drv}" \
+      --owner=0 --group=0 --mode=u+rw,uga+r --hard-dereference . \
+      | gzip -9 > $out
+  '';
+  bundle-win-exe = drv: callPackage ./bundle-win-exe.nix { cronosd = drv; };
+  matrix = lib.cartesianProductOfSets {
+    network = [ "mainnet" "testnet" ];
+    pkgtype = [
+      "nix" # normal nix package
+      "bundle" # relocatable bundled package
+      "tarball" # tarball of the bundle, for distribution and checksum
+    ];
+  };
+in
+builtins.listToAttrs (builtins.map
+  ({ network, pkgtype }: {
+    name = builtins.concatStringsSep "-" (
+      [ "cronosd" ] ++
+      lib.optional (network != "mainnet") network ++
+      lib.optional (pkgtype != "nix") pkgtype
+    );
+    value =
+      let
+        cronosd = callPackage ../. {
+          inherit rev network;
+        };
+        bundle =
+          if stdenv.hostPlatform.isWindows then
+            bundle-win-exe cronosd
+          else
+            bundle-exe cronosd;
+      in
+      if pkgtype == "bundle" then
+        bundle
+      else if pkgtype == "tarball" then
+        make-tarball bundle
+      else
+        cronosd;
+  })
+  matrix
+)

nix/testground-image.nix

@@ -0,0 +1,22 @@
+{ dockerTools, runCommandLocal, cronos-matrix, benchmark-testcase }:
+let
+  tmpDir = runCommandLocal "tmp" { } ''
+    mkdir -p $out/tmp/
+  '';
+in
+dockerTools.buildLayeredImage {
+  name = "cronos-testground";
+  created = "now";
+  contents = [
+    benchmark-testcase
+    cronos-matrix.cronosd
+    tmpDir
+  ];
+  config = {
+    Expose = [ 9090 26657 26656 1317 26658 26660 26659 30000 ];
+    Cmd = [ "/bin/stateless-testcase" ];
+    Env = [
+      "PYTHONUNBUFFERED=1"
+    ];
+  };
+}

scripts/release.sh

@@ -28,7 +28,7 @@ build() {
         fi
     fi
     echo "building $FLAKE"
-    nix build -L "$FLAKE"
+    nix build --no-update-lock-file --no-allow-dirty -L "$FLAKE"
     cp result "cronos_${ref_name_clean:1}${network}_${name}.tar.gz"
 }
 

testground/README.md

@@ -0,0 +1,77 @@
+# Testground
+
+The implementation is inspired by [testground](https://github.com/testground/testground), but we did a lot of simplifications to make it easier to deploy:
+
+- No centralized sync service, each node are assigned an unique continuous integer identifier, and node's hostname can be derived from that, that's how nodes discover each other and build the network.
+- Don't support networking configuration, but we might implement it in the future.
+
+## Build Image
+
+>  Prerequisites: nix, for macOS also need [linux remote builder](https://nix.dev/manual/nix/2.22/advanced-topics/distributed-builds.html)
+
+You can test with the prebuilt images in [github registry](https://github.com/crypto-org-chain/cronos/pkgs/container/cronos-testground), or build the image locally:
+
+```bash
+$ nix build .#testground-image
+# for apple silicon mac: nix build .#legacyPackages.aarch64-linux.testground-image
+# for x86 mac: nix build .#legacyPackages.x86_64-linux.testground-image
+$ docker load < ./result
+Loaded image: cronos-testground:<imageID>
+$ docker tag cronos-testground:<imageID> ghcr.io/crypto-org-chain/cronos-testground:latest
+```
+
+Or one liner like this:
+
+```bash
+docker load < $(nix build .#legacyPackages.aarch64-linux.testground-image --no-link --print-out-paths) \
+  | grep "^Loaded image:" \
+  | cut -d ' ' -f 3 \
+  | xargs -I{} docker tag {} ghcr.io/crypto-org-chain/cronos-testground:latest
+```
+
+## Generate data files locally
+
+You need to have the `cronosd` in `PATH`.
+
+```bash
+nix run .#stateless-testcase -- gen /tmp/data/out \
+  --validator-generate-load \
+  --validators 3 \
+  --fullnodes 0 \
+  --num-accounts 800 \
+  --num-txs 20 \
+  --app-patch '{"mempool": {"max-txs": -1}}' \
+  --config-patch '{"mempool": {"size": 100000}}' \
+  --tx-type erc20-transfer \
+  --genesis-patch '{"consensus_params": {"block": {"max_gas": "263000000"}}}'
+```
+
+* `validators`/`fullnodes` is the number of validators/full nodes.
+* `num_accounts` is the number of test accounts for each full node.
+* `num_txs` is the number of test transactions to be sent for each test account.
+* `config`/`app` is the config patch for config/app.toml.
+* `genesis` is the patch for genesis.json.
+
+## Embed the data directory
+
+Embed the data directory into the image, it produce a new image:
+
+```bash
+$ nix run github:crypto-org-chain/cronos#stateless-testcase patchimage cronos-testground:latest /tmp/data/out
+```
+
+## Run With Docker Compose
+
+```bash
+$ mkdir /tmp/outputs
+$ jsonnet -S testground/benchmark/compositions/docker-compose.jsonnet \
+  --ext-str outputs=/tmp/colima \
+  --ext-code nodes=3 \
+  | docker-compose -f /dev/stdin up --remove-orphans --force-recreate
+```
+
+It'll collect the node data files to the `/tmp/outputs` directory.
+
+## Run In Cluster
+
+Please use [cronos-testground](https://github.com/crypto-org-chain/cronos-testground) to run the benchmark in k8s cluster.