[SPARK-49527] Add ConfOptionDocGenerator to generate Spark Operator Config Property Doc

### What changes were proposed in this pull request?

This PR adds a module `docs-utils` to automatically generate config properties doc page from source.

### Why are the changes needed?

This helps to keep config property docs up-to-date for properties by adding it as a gradle task.

### Does this PR introduce _any_ user-facing change?

No (doc only, not released)

### How was this patch tested?

Pass the CIs

### Was this patch authored or co-authored using generative AI tooling?

No

Closes #118 from jiangzho/doc_utils.

Lead-authored-by: zhou-jiang <[email protected]>
Co-authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

Author: jiangzho

build-tools/docs-utils/build.gradle

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ext {
+    javaMainClass = "org.apache.spark.k8s.operator.utils.ConfOptionDocGenerator"
+    docsPath = System.getProperty("user.dir") + "/docs"
+}
+
+dependencies {
+    implementation project(":spark-operator")
+    implementation(libs.log4j.core)
+    implementation(libs.log4j.slf4j.impl)
+    compileOnly(libs.lombok)
+    annotationProcessor(libs.lombok)
+}
+
+test {
+    useJUnitPlatform()
+}
+
+tasks.register('generateConfPropsDoc', Exec) {
+    description = "Generate config properties doc for operator"
+    commandLine "java", "-classpath", sourceSets.main.runtimeClasspath.getAsPath(), javaMainClass, docsPath
+}

build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/ConfOptionDocGenerator.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.k8s.operator.utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.reflect.Field;
+import java.util.List;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.spark.k8s.operator.config.ConfigOption;
+import org.apache.spark.k8s.operator.config.SparkOperatorConf;
+
+@Slf4j
+public class ConfOptionDocGenerator {
+  public static final String CONF_FILE_NAME = "config_properties.md";
+  public static final String DEFAULT_DOCS_PATH = "docs";
+  public static final String GENERATED_FILE_HEADER =
+      "This doc is automatically generated by gradle task, manual updates would be overridden.";
+
+  public void generate(String docsPath) throws IOException, IllegalAccessException {
+    Field[] fields = SparkOperatorConf.class.getDeclaredFields();
+    File docsDir = new File(docsPath);
+    if (!docsDir.exists() && docsDir.mkdir()) {
+      log.info("Creating docs directory at {}", docsPath);
+    }
+    File generated = new File(docsPath, CONF_FILE_NAME);
+    if (generated.createNewFile()) {
+      log.info("Creating props at {}/{}", docsPath, CONF_FILE_NAME);
+    }
+    PrintWriter printWriter = new PrintWriter(generated, "UTF-8");
+    printWriter.println(String.format("[//]: # (%s)", GENERATED_FILE_HEADER));
+    printWriter.println("# Spark Operator Config Properties");
+    DocTable table =
+        DocTable.builder()
+            .headers(List.of("Key", "Type", "Default Value", "Allow Hot Reloading", "Description"))
+            .columns(5)
+            .build();
+    for (Field f : fields) {
+      if (ConfigOption.class.isAssignableFrom(f.getType())) {
+        ConfigOption<?> conf = (ConfigOption<?>) f.get(this);
+        table.addRow(
+            List.of(
+                conf.getKey(),
+                conf.getTypeParameterClass().getSimpleName(),
+                conf.getDefaultValue().toString(),
+                String.valueOf(conf.isEnableDynamicOverride()),
+                conf.getDescription()));
+      }
+    }
+    table.flush(printWriter);
+    printWriter.close();
+  }
+
+  public static void main(String[] args) throws IOException, IllegalAccessException {
+    ConfOptionDocGenerator generator = new ConfOptionDocGenerator();
+    String docsPath = DEFAULT_DOCS_PATH;
+    if (args.length > 0) {
+      docsPath = args[0];
+    }
+    try {
+      generator.generate(docsPath);
+    } catch (IOException | IllegalAccessException e) {
+      log.error("Failed to generate docs for config props.", e);
+      throw e;
+    }
+  }
+}

build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/DocTable.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.k8s.operator.utils;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+@Data
+@RequiredArgsConstructor
+@Builder
+public class DocTable {
+  // Separator among cells
+  public static final String ROW_SEPARATOR = " | ";
+  // Separator for header line
+  public static final String HEADER_SEPARATOR = "---";
+  private final List<String> headers;
+  private final int columns;
+  @Builder.Default private final List<List<String>> rows = new ArrayList<>();
+
+  public void addRow(List<String> row) {
+    rows.add(row);
+  }
+
+  public void flush(PrintWriter writer) {
+    writer.println(joinRow(headers));
+    writer.println(joinRow(Collections.nCopies(columns, HEADER_SEPARATOR)));
+    for (List<String> row : rows) {
+      writer.println(joinRow(row));
+    }
+    writer.println();
+    writer.flush();
+  }
+
+  private String joinRow(List<String> elements) {
+    StringBuilder stringBuilder = new StringBuilder(ROW_SEPARATOR);
+    for (String element : elements) {
+      stringBuilder.append(element);
+      stringBuilder.append(ROW_SEPARATOR);
+    }
+    if (elements.size() < columns) {
+      // Append empty cells to end if needed
+      stringBuilder.append(ROW_SEPARATOR.repeat(columns - elements.size()));
+    }
+    return stringBuilder.toString();
+  }
+}

settings.gradle

@@ -20,3 +20,5 @@ rootProject.name = 'apache-spark-kubernetes-operator'
 include 'spark-operator-api'
 include 'spark-submission-worker'
 include 'spark-operator'
+include "build-tools-docs-utils"
+project(':build-tools-docs-utils').projectDir = file('build-tools/docs-utils')

spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java

@@ -42,8 +42,8 @@ public class ConfigOption<T> {
   @Getter @Builder.Default private final boolean enableDynamicOverride = true;
   @Getter private String key;
   @Getter private String description;
-  private T defaultValue;
-  private Class<T> typeParameterClass;
+  @Getter private T defaultValue;
+  @Getter private Class<T> typeParameterClass;
 
   public T getValue() {
     T resolvedValue = resolveValue();