[CELEBORN-1699] Add GA to check the openapi codegen

.github/workflows/sbt.yml

@@ -260,3 +260,22 @@ jobs:
             **/target/test/
             **/target/test-reports/**
             **/target/unit-tests.log
+
+  openapi-codegen-check:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        java:
+          - 11
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: zulu
+          java-version: ${{ matrix.java }}
+          check-latest: false
+      - name: Test with SBT
+        run: |
+          build/sbt "clean;celeborn-openapi-client/check"

openapi/openapi-client/src/main/openapi3/master_rest_v1.yaml

@@ -583,6 +583,13 @@ components:
         inNative:
           type: boolean
           description: Whether the thread is executing native code.
+        isDaemon:
+          type: boolean
+          description: Whether the thread is a daemon thread.
+        priority:
+          type: integer
+          format: int32
+          description: The priority of the thread.
       required:
         - threadId
         - threadName

openapi/openapi-client/src/main/openapi3/worker_rest_v1.yaml

@@ -320,6 +320,13 @@ components:
         inNative:
           type: boolean
           description: Whether the thread is executing native code.
+        isDaemon:
+          type: boolean
+          description: Whether the thread is a daemon thread.
+        priority:
+          type: integer
+          format: int32
+          description: The priority of the thread.
       required:
         - threadId
         - threadName

project/CelebornBuild.scala

@@ -1356,6 +1356,7 @@ object CelebornOpenApi {
   val openApiClientOutputDir = "openapi/openapi-client/src/main/java"
 
   val generate = TaskKey[Unit]("generate", "generate openapi client code")
+  val check = TaskKey[Unit]("check", "check the openapi spec and generated code")
 
   val commonOpenApiClientGenerateSettings = Seq(
     openApiGeneratorName := "java",
@@ -1432,6 +1433,50 @@ object CelebornOpenApi {
         IO.copyDirectory(workerSrcDir, dstDir)
       },
 
+      check := {
+        (openApiClientMasterGenerate / Compile / openApiGenerate).value
+        (openApiClientWorkerGenerate / Compile / openApiGenerate).value
+
+        val internalMasterSrcDir = file(openApiMasterInternalOutputDir) / "src" / "main" / "java"
+        val internalWorkerSrcDir = file(openApiWorkerInternalOutputDir) / "src" / "main" / "java"
+        val openApiSrcDir = file(openApiClientOutputDir)
+
+        def getRelativePaths(dir: File): Set[String] = {
+          (dir ** "*.java").get.map(_.relativeTo(dir).get.getPath).toSet
+        }
+        val internalSrcPaths = getRelativePaths(internalMasterSrcDir) ++ getRelativePaths(internalWorkerSrcDir)
+        val openApiSrcPaths = getRelativePaths(openApiSrcDir)
+        val notGeneratedSrcPaths = openApiSrcPaths -- internalSrcPaths
+        if (notGeneratedSrcPaths.nonEmpty) {
+          sys.error(s"Files ${notGeneratedSrcPaths.mkString(", ")} not generated by openapi generator anymore, seems outdated.")
+        }
+
+        def diffDirSrcFiles(srcDir: File, dstDir: File): Unit = {
+          val srcFiles = (srcDir ** "*.java").get
+          val dstFiles = (dstDir ** "*.java").get
+
+          srcFiles.foreach { srcFile =>
+            val relativePath = srcFile.relativeTo(srcDir).get.getPath
+            val dstFile = dstDir / relativePath
+
+            if (!dstFile.exists()) {
+              sys.error(s"File $relativePath does not exist in the openapi client code directory")
+            } else {
+              val srcContent = IO.read(srcFile, UTF_8)
+              val dstContent = IO.read(dstFile, UTF_8)
+
+              if (srcContent != dstContent) {
+                sys.error(s"File $relativePath differs, please re-generate the code.")
+              }
+            }
+          }
+        }
+        diffDirSrcFiles(internalMasterSrcDir, openApiSrcDir)
+        diffDirSrcFiles(internalWorkerSrcDir, openApiSrcDir)
+
+        streams.value.log.info("The openapi spec and code are consistent.")
+      },
+
       (assembly / test) := { },
       (assembly / assemblyJarName) := {
         s"${moduleName.value}-${version.value}.${artifact.value.extension}"