#3967 refactor ha and load tests

.github/workflows/e2e-ha.yml → .github/workflows/ha-resilience-tests.yml

@@ -1,13 +1,9 @@
-name: E2E HA Tests
+name: HA Resilence Tests
 
 on:
   workflow_dispatch:
   schedule:
     - cron: "0 0 * * *" # Runs daily at midnight
-  pull_request:
-    branches:
-      - main
-
 
 jobs:
   setup:

.github/workflows/load-tests.yml

@@ -0,0 +1,66 @@
+name: Load Tests
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *" # Runs daily at midnight
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      packages: write
+      attestations: write
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Ensure SHA pinned actions
+        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@471d5ace1f08e3c4df1c4c2f7e6341aa75da434a # v5.0.3
+      - name: Run pre-commit
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.13.0"
+          cache: "pip"
+      - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+        with:
+          distribution: "temurin"
+          java-version: 21
+
+      - name: Cache local Maven repository
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+
+      - name: Build and package with Maven Docker profile
+        run: ./mvnw clean install -Pdocker -DskipTests --batch-mode --errors  --show-version
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Load Tests
+        run: ./mvnw verify -Pintegration --batch-mode --errors --fail-never --show-version -pl load-tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Tests Reporter
+        uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0
+        if: success() || failure()
+        with:
+          name: IT Tests Report
+          path: "**/failsafe-reports/TEST*.xml"
+          list-tests: "failed"
+          list-suites: "failed"
+          reporter: java-junit

.github/workflows/mvn-test.yml

@@ -325,50 +325,6 @@ jobs:
           list-tests: "failed"
           reporter: java-junit
 
-  java-load-tests:
-    runs-on: ubuntu-latest
-    needs: build-and-package
-    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Set up JDK 21
-        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
-        with:
-          distribution: "temurin"
-          java-version: 21
-          cache: "maven"
-
-      - name: Restore Maven artifacts
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
-        with:
-          path: ~/.m2/repository
-          key: maven-repo-${{ github.run_id }}-${{ github.run_attempt }}
-
-      - name: Restore Docker image
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
-        with:
-          path: /tmp/arcadedb-image.tar
-          key: docker-image-${{ github.run_id }}-${{ github.run_attempt }}
-
-      - name: Load Docker image
-        run: docker load < /tmp/arcadedb-image.tar
-
-      - name: E2E Perf Tests
-        run: ./mvnw verify -Pintegration -pl load-tests
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          ARCADEDB_DOCKER_IMAGE: ${{ needs.build-and-package.outputs.image-tag }}
-
-      - name: E2E Perf Tests Reporter
-        uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0
-        if: success() || failure()
-        with:
-          name: Java Load Tests Report
-          path: "load-tests/target/failsafe-reports/TEST*.xml"
-          list-suites: "failed"
-          list-tests: "failed"
-          reporter: java-junit
-
   js-e2e-tests:
     runs-on: ubuntu-latest
     needs: build-and-package

load-tests/src/test/java/com/arcadedb/test/load/ThreeINodesLoadtTestIT.java

@@ -0,0 +1,162 @@
+/*
+ * Copyright 2021-present Arcade Data Ltd (info@arcadedata.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package com.arcadedb.test.load;
+
+import com.arcadedb.test.support.ContainersTestTemplate;
+import com.arcadedb.test.support.DatabaseWrapper;
+import com.arcadedb.test.support.ServerWrapper;
+import io.micrometer.core.instrument.Metrics;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+class ThreeINodesLoadtTestIT extends ContainersTestTemplate {
+
+  private static final String SERVER_LIST = "arcadedb-0:2434:2480,arcadedb-1:2434:2480,arcadedb-2:2434:2480";
+
+  @AfterEach
+  @Override
+  public void tearDown() {
+    // Skip compareAllDatabases(): with non-persistent containers, database files are not
+    // on the host after stop. The test body already verifies convergence via Awaitility.
+    super.tearDown();
+  }
+
+  @ParameterizedTest(name = "Three-node Raft HA Load test with {0} protocol")
+  @EnumSource(DatabaseWrapper.Protocol.class)
+  @DisplayName("Three-node Raft HA: replication across all nodes with consistency check")
+  void threeNodeReplication(DatabaseWrapper.Protocol protocol) {
+    createArcadeContainer("arcadedb-0", SERVER_LIST, "majority", network);
+    createArcadeContainer("arcadedb-1", SERVER_LIST, "majority", network);
+    createArcadeContainer("arcadedb-2", SERVER_LIST, "majority", network);
+
+    logger.info("Starting all containers");
+    final List<ServerWrapper> servers = startCluster();
+
+    final DatabaseWrapper db1 = new DatabaseWrapper(servers.get(0), idSupplier, wordSupplier);
+    final DatabaseWrapper db2 = new DatabaseWrapper(servers.get(1), idSupplier, wordSupplier);
+    final DatabaseWrapper db3 = new DatabaseWrapper(servers.get(2), idSupplier, wordSupplier);
+
+    logger.info("Creating database and schema");
+    db1.createDatabase();
+    db1.createSchema();
+
+    logger.info("Checking schema replicated to all nodes");
+    db1.checkSchema();
+    db2.checkSchema();
+    db3.checkSchema();
+
+    final int numOfThreads = 3; //number of threads to use to insert users and photos
+    final int numOfUsers = 1000; // Each thread will create 200000 users
+    final int numOfPhotos = 10; // Each user will have 5 photos
+    final int numOfFriendship = 0; // Each thread will create 100000 friendships
+    final int numOfLike = 0; // Each thread will create 100000 likes
+
+    int expectedUsersCount = numOfUsers * numOfThreads;
+    int expectedPhotoCount = expectedUsersCount * numOfPhotos;
+    int expectedFriendshipCount = numOfFriendship;
+    int expectedLikeCount = numOfLike;
+    LocalDateTime startedAt = LocalDateTime.now();
+    logger.info("Starting load test on protocol {}", protocol.name());
+    logger.info("Creating {} users using {} threads", expectedUsersCount, numOfThreads);
+    logger.info("Expected users: {} - photos: {} - friendships: {} - likes: {}", expectedUsersCount, expectedPhotoCount,
+        expectedFriendshipCount, expectedLikeCount);
+    logger.info("Starting at {}", DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(startedAt));
+
+    ExecutorService executor = Executors.newFixedThreadPool(10);
+    for (int i = 0; i < numOfThreads; i++) {
+      // Each thread will create users and photos
+      executor.submit(() -> {
+        DatabaseWrapper db = new DatabaseWrapper(servers.getFirst(), idSupplier, wordSupplier, protocol);
+        db.addUserAndPhotos(numOfUsers, numOfPhotos);
+        db.close();
+      });
+    }
+    // Each thread will create friendships
+    executor.submit(() -> {
+      DatabaseWrapper db = new DatabaseWrapper(servers.getFirst(), idSupplier, wordSupplier, protocol);
+      db.createFriendships(numOfFriendship);
+      db.close();
+    });
+    // Each thread will create friendships
+    executor.submit(() -> {
+      DatabaseWrapper db = new DatabaseWrapper(servers.getFirst(), idSupplier, wordSupplier, protocol);
+      db.createLike(numOfLike);
+      db.close();
+    });
+
+    executor.shutdown();
+
+    while (!executor.isTerminated()) {
+      try {
+        final long users1 = db1.countUsers();
+        final long photos1 = db1.countPhotos();
+        final long users2 = db2.countUsers();
+        final long photos2 = db2.countPhotos();
+        final long users3 = db3.countUsers();
+        final long photos3 = db3.countPhotos();
+        logger.info("Users: {} / {} / {} | Photos: {} / {} / {}", users1, users2, users3, photos1, photos2, photos3);
+
+      } catch (Exception e) {
+        logger.error(e.getMessage(), e);
+      }
+      try {
+        // Wait for 2 seconds before checking again
+        TimeUnit.SECONDS.sleep(5);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+      }
+    }
+    LocalDateTime finishedAt = LocalDateTime.now();
+    logger.info("Finishing at {}", DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(finishedAt));
+    logger.info("Total time: {} minutes", Duration.between(startedAt, finishedAt).toMinutes());
+
+    Metrics.globalRegistry.getMeters().forEach(meter -> {
+      logger.info("Meter: {} - {}", meter.getId().getName(), meter.measure());
+    });
+
+    db1.assertThatUserCountIs(expectedUsersCount);
+    db1.assertThatPhotoCountIs(expectedPhotoCount);
+    db1.assertThatFriendshipCountIs(expectedFriendshipCount);
+    db1.assertThatLikesCountIs(expectedLikeCount);
+
+    db2.assertThatUserCountIs(expectedUsersCount);
+    db2.assertThatPhotoCountIs(expectedPhotoCount);
+    db2.assertThatFriendshipCountIs(expectedFriendshipCount);
+    db2.assertThatLikesCountIs(expectedLikeCount);
+
+    db3.assertThatUserCountIs(expectedUsersCount);
+    db3.assertThatPhotoCountIs(expectedPhotoCount);
+    db3.assertThatFriendshipCountIs(expectedFriendshipCount);
+    db3.assertThatLikesCountIs(expectedLikeCount);
+
+    db1.close();
+    db2.close();
+    db3.close();
+  }
+}