diff --git a/build-tools/docs-utils/build.gradle b/build-tools/docs-utils/build.gradle new file mode 100644 index 00000000..2cdde290 --- /dev/null +++ b/build-tools/docs-utils/build.gradle @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +ext { + javaMainClass = "org.apache.spark.k8s.operator.utils.ConfOptionDocGenerator" + docsPath = System.getProperty("user.dir") + "/docs" +} + +dependencies { + implementation project(":spark-operator") + implementation(libs.log4j.core) + implementation(libs.log4j.slf4j.impl) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) +} + +test { + useJUnitPlatform() +} + +tasks.register('generateConfPropsDoc', Exec) { + description = "Generate config properties doc for operator" + commandLine "java", "-classpath", sourceSets.main.runtimeClasspath.getAsPath(), javaMainClass, docsPath +} diff --git a/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/ConfOptionDocGenerator.java b/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/ConfOptionDocGenerator.java new file mode 100644 index 00000000..c0f2f497 --- /dev/null +++ b/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/ConfOptionDocGenerator.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.k8s.operator.utils; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.reflect.Field; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.spark.k8s.operator.config.ConfigOption; +import org.apache.spark.k8s.operator.config.SparkOperatorConf; + +@Slf4j +public class ConfOptionDocGenerator { + public static final String CONF_FILE_NAME = "config_properties.md"; + public static final String DEFAULT_DOCS_PATH = "docs"; + public static final String GENERATED_FILE_HEADER = + "This doc is automatically generated by gradle task, manual updates would be overridden."; + + public void generate(String docsPath) throws IOException, IllegalAccessException { + Field[] fields = SparkOperatorConf.class.getDeclaredFields(); + File docsDir = new File(docsPath); + if (!docsDir.exists() && docsDir.mkdir()) { + log.info("Creating docs directory at {}", docsPath); + } + File generated = new File(docsPath, CONF_FILE_NAME); + if (generated.createNewFile()) { + log.info("Creating props at {}/{}", docsPath, CONF_FILE_NAME); + } + PrintWriter printWriter = new PrintWriter(generated, "UTF-8"); + printWriter.println(String.format("[//]: # (%s)", GENERATED_FILE_HEADER)); + printWriter.println("# Spark Operator Config Properties"); + DocTable table = + DocTable.builder() + .headers(List.of("Key", "Type", "Default Value", "Allow Hot Reloading", "Description")) + .columns(5) + .build(); + for (Field f : fields) { + if (ConfigOption.class.isAssignableFrom(f.getType())) { + ConfigOption conf = (ConfigOption) f.get(this); + table.addRow( + List.of( + conf.getKey(), + conf.getTypeParameterClass().getSimpleName(), + conf.getDefaultValue().toString(), + String.valueOf(conf.isEnableDynamicOverride()), + conf.getDescription())); + } + } + table.flush(printWriter); + printWriter.close(); + } + + public static void main(String[] args) throws IOException, IllegalAccessException { + ConfOptionDocGenerator generator = new ConfOptionDocGenerator(); + String docsPath = DEFAULT_DOCS_PATH; + if (args.length > 0) { + docsPath = args[0]; + } + try { + generator.generate(docsPath); + } catch (IOException | IllegalAccessException e) { + log.error("Failed to generate docs for config props.", e); + throw e; + } + } +} diff --git a/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/DocTable.java b/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/DocTable.java new file mode 100644 index 00000000..73e0597d --- /dev/null +++ b/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/DocTable.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.k8s.operator.utils; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import lombok.Builder; +import lombok.Data; +import lombok.RequiredArgsConstructor; + +@Data +@RequiredArgsConstructor +@Builder +public class DocTable { + // Separator among cells + public static final String ROW_SEPARATOR = " | "; + // Separator for header line + public static final String HEADER_SEPARATOR = "---"; + private final List headers; + private final int columns; + @Builder.Default private final List> rows = new ArrayList<>(); + + public void addRow(List row) { + rows.add(row); + } + + public void flush(PrintWriter writer) { + writer.println(joinRow(headers)); + writer.println(joinRow(Collections.nCopies(columns, HEADER_SEPARATOR))); + for (List row : rows) { + writer.println(joinRow(row)); + } + writer.println(); + writer.flush(); + } + + private String joinRow(List elements) { + StringBuilder stringBuilder = new StringBuilder(ROW_SEPARATOR); + for (String element : elements) { + stringBuilder.append(element); + stringBuilder.append(ROW_SEPARATOR); + } + if (elements.size() < columns) { + // Append empty cells to end if needed + stringBuilder.append(ROW_SEPARATOR.repeat(columns - elements.size())); + } + return stringBuilder.toString(); + } +} diff --git a/settings.gradle b/settings.gradle index 8b2b816a..bf5289fb 100644 --- a/settings.gradle +++ b/settings.gradle @@ -20,3 +20,5 @@ rootProject.name = 'apache-spark-kubernetes-operator' include 'spark-operator-api' include 'spark-submission-worker' include 'spark-operator' +include "build-tools-docs-utils" +project(':build-tools-docs-utils').projectDir = file('build-tools/docs-utils') diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java index 83091bd9..e0726c4f 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java @@ -42,8 +42,8 @@ public class ConfigOption { @Getter @Builder.Default private final boolean enableDynamicOverride = true; @Getter private String key; @Getter private String description; - private T defaultValue; - private Class typeParameterClass; + @Getter private T defaultValue; + @Getter private Class typeParameterClass; public T getValue() { T resolvedValue = resolveValue();