HDDS-10573. [hsync] Add a freon tool to benchmark hsync/write concurrency.

.github/dependabot.yml

@@ -14,29 +14,14 @@
 # limitations under the License.
 version: 2
 updates:
-  # Maintain dependencies for GitHub Actions
-  - package-ecosystem: "npm"
-    directory: "hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/"
-    schedule:
-      interval: "weekly"
-    commit-message:
-      prefix: "[Recon] Dependabot Package Upgrade: "
-    groups:
-      minor-updates:
-        patterns:
-          - "*"
-        update-types:
-          - "minor"
-          - "patch"
+  - package-ecosystem: maven
+    directory: "/"
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-major"]
-    pull-request-branch-name:
-      separator: "-"
-
-  - package-ecosystem: maven
-    directory: "/"
     schedule:
       interval: "weekly"
+      day: "saturday"
+      time: "07:00" # UTC
     pull-request-branch-name:
       separator: "-"

hadoop-ozone/dist/src/main/smoketest/env-compose.robot → .github/labeler.yml

@@ -13,20 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-*** Settings ***
-Documentation       High level utilities to execute commands and tests in docker-compose based environments.
-Resource            commonlib.robot
-Test Timeout        5 minutes
+# Configuration for .github/workflows/label-pr.yml
 
-*** Keywords ***
+# This rule can be deleted once the container reconciliation feature branch is merged.
+container-reconciliation:
+- base-branch: HDDS-10239-container-reconciliation
 
-Run tests on host
-    [arguments]        ${host}       ${robotfile}
-    ${result} =        Execute       docker-compose exec ${host} robot smoketest/${robotfile}
-
-Execute on host
-    [arguments]                     ${host}     ${command}
-    ${rc}                           ${output} =                 Run And Return Rc And Output           docker-compose exec ${host} ${command}
-    Log                             ${output}
-    Should Be Equal As Integers     ${rc}                       0
-    [return]                        ${output}

.github/workflows/build-ratis.yml

@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This workflow can be called by other workflows to build Ratis.
+#
+# Inputs:
+# - Ratis repo
+# - the commit to build
+# Outputs:
+# - various version numbers that need to be provided to the Ozone build process.
+# - Ratis repository is uploaded as an artifact named `ratis-jars`
+#
+# See `intermittent-test-check.yml` as an example use of this workflow.
+
+name: build-ratis
+on:
+  workflow_call:
+    inputs:
+      repo:
+        description: Ratis repository
+        default: apache/ratis
+        required: true
+        type: string
+      ref:
+        description: Ratis ref (branch, tag or commit SHA)
+        default: master
+        required: true
+        type: string
+    outputs:
+      ratis-version:
+        description: "Ratis Version"
+        value: ${{ jobs.ratis.outputs.ratis-version }}
+      thirdparty-version:
+        description: "Ratis Third-Party Version"
+        value: ${{ jobs.ratis.outputs.thirdparty-version }}
+      grpc-version:
+        description: "gRPC Version"
+        value: ${{ jobs.ratis-thirdparty.outputs.grpc-version }}
+      netty-version:
+        description: "Netty Version"
+        value: ${{ jobs.ratis-thirdparty.outputs.netty-version }}
+      protobuf-version:
+        description: "Protobuf Version"
+        value: ${{ jobs.ratis-thirdparty.outputs.protobuf-version }}
+env:
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+jobs:
+  ratis:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    outputs:
+      ratis-version: ${{ steps.versions.outputs.ratis }}
+      thirdparty-version: ${{ steps.versions.outputs.thirdparty }}
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.repo }}
+          ref: ${{ inputs.ref }}
+      - name: Cache for maven dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository
+            !~/.m2/repository/org/apache/ratis
+          key: ratis-dependencies-${{ hashFiles('**/pom.xml') }}
+      - name: Setup java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 8
+      - name: Get component versions
+        id: versions
+        run: |
+          thirdparty_version="$(mvn help:evaluate -N -q -DforceStdout -Dexpression=ratis.thirdparty.version)"
+          echo "thirdparty=${thirdparty_version}" >> $GITHUB_OUTPUT
+
+          ratis_sha=$(git rev-parse --short HEAD)
+          ratis_version="$(mvn help:evaluate -N -q -DforceStdout -Dexpression=project.version | sed -e "s/-SNAPSHOT/-${ratis_sha}-SNAPSHOT/")"
+          echo "ratis=${ratis_version}" >> $GITHUB_OUTPUT
+      - name: Run a full build
+        run: |
+          mvn versions:set -DnewVersion=${{ steps.versions.outputs.ratis }}
+          dev-support/checks/build.sh
+      - name: Store Maven repo for tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: ratis-jars
+          path: |
+            ~/.m2/repository/org/apache/ratis
+          retention-days: 1
+  ratis-thirdparty:
+    runs-on: ubuntu-20.04
+    needs:
+      - ratis
+    timeout-minutes: 30
+    outputs:
+      grpc-version: ${{ steps.versions.outputs.grpc }}
+      netty-version: ${{ steps.versions.outputs.netty }}
+      protobuf-version: ${{ steps.versions.outputs.protobuf }}
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+        with:
+          repository: apache/ratis-thirdparty
+          ref: ${{ needs.ratis.outputs.thirdparty-version }}
+      - name: Get component versions
+        id: versions
+        run: |
+          echo "grpc=$(mvn help:evaluate -N -q -DforceStdout -Dexpression=shaded.grpc.version)" >> $GITHUB_OUTPUT
+          echo "netty=$(mvn help:evaluate -N -q -DforceStdout -Dexpression=shaded.netty.version)" >> $GITHUB_OUTPUT
+          echo "protobuf=$(mvn help:evaluate -N -q -DforceStdout -Dexpression=shaded.protobuf.version)" >> $GITHUB_OUTPUT
+  debug:
+    runs-on: ubuntu-20.04
+    needs:
+      - ratis
+      - ratis-thirdparty
+    steps:
+      - name: Print versions
+        run: |
+          echo ${{ needs.ratis.outputs.ratis-version }}
+          echo ${{ needs.ratis.outputs.thirdparty-version }}
+          echo ${{ needs.ratis-thirdparty.outputs.grpc-version }}
+          echo ${{ needs.ratis-thirdparty.outputs.netty-version }}
+          echo ${{ needs.ratis-thirdparty.outputs.protobuf-version }}

.github/workflows/ci.yml

@@ -27,9 +27,8 @@ jobs:
     outputs:
       acceptance-suites: ${{ steps.acceptance-suites.outputs.suites }}
       needs-basic-check: ${{ steps.categorize-basic-checks.outputs.needs-basic-check }}
-      needs-unit-check: ${{ steps.categorize-basic-checks.outputs.needs-unit-check }}
+      needs-native-check: ${{ steps.categorize-basic-checks.outputs.needs-native-check }}
       basic-checks: ${{ steps.categorize-basic-checks.outputs.basic-checks }}
-      unit-checks: ${{ steps.categorize-basic-checks.outputs.unit-checks }}
       needs-build: ${{ steps.selective-checks.outputs.needs-build }}
       needs-compile: ${{ steps.selective-checks.outputs.needs-compile }}
       needs-compose-tests: ${{ steps.selective-checks.outputs.needs-compose-tests }}
@@ -92,10 +91,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pnpm-
       - name: Cache for maven dependencies
-        uses: actions/cache@v4
+        uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
@@ -164,7 +163,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
@@ -202,7 +201,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
@@ -228,25 +227,21 @@ jobs:
           name: ${{ matrix.check }}
           path: target/${{ matrix.check }}
         continue-on-error: true
-  unit:
+  native:
     needs:
       - build-info
       - basic
     runs-on: ubuntu-20.04
     timeout-minutes: 150
-    if: needs.build-info.outputs.needs-unit-check == 'true'
-    strategy:
-      matrix:
-        check: ${{ fromJson(needs.build-info.outputs.unit-checks) }}
-      fail-fast: false
+    if: needs.build-info.outputs.needs-native-check == 'true'
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
       - name: Cache for maven dependencies
         uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
@@ -257,19 +252,19 @@ jobs:
           distribution: 'temurin'
           java-version: 8
       - name: Execute tests
-        run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh
+        run: hadoop-ozone/dev-support/checks/${{ github.job }}.sh
         continue-on-error: true
         env:
           GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Summary of failures
-        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ matrix.check }}/summary.txt
+        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
         if: ${{ !cancelled() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
         with:
-          name: ${{ matrix.check }}
-          path: target/${{ matrix.check }}
+          name: ${{ github.job }}
+          path: target/${{ github.job }}
         continue-on-error: true
   dependency:
     needs:
@@ -314,7 +309,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
@@ -434,12 +429,14 @@ jobs:
       matrix:
         profile:
           - client
-          - contract
+          - container
           - filesystem
           - hdds
           - om
           - ozone
+          - recon
           - shell
+          - snapshot
           - flaky
       fail-fast: false
     steps:
@@ -449,31 +446,19 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             maven-repo-
-      - name: Download Ozone repo
-        id: download-ozone-repo
-        uses: actions/download-artifact@v4
-        with:
-          name: ozone-repo
-          path: |
-            ~/.m2/repository/org/apache/ozone
-        continue-on-error: true
       - name: Setup java
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: 8
+          java-version: 17
       - name: Execute tests
         continue-on-error: true
         run: |
-          if [[ -e "${{ steps.download-ozone-repo.outputs.download-path }}" ]]; then
-            export OZONE_REPO_CACHED=true
-          fi
-
           args=
           if [[ "${{ matrix.profile }}" == "flaky" ]]; then
             args="-Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600"
@@ -497,9 +482,9 @@ jobs:
     timeout-minutes: 30
     if: github.repository == 'apache/ozone' && github.event_name != 'pull_request'
     needs:
-      - unit
       - acceptance
       - integration
+      - native
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
@@ -509,7 +494,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |