diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 4df7d0deb6db0..c871253aed6e4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -40,6 +40,11 @@ import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.ValidationUtils; +import org.apache.hudi.config.metrics.HoodieMetricsConfig; +import org.apache.hudi.config.metrics.HoodieMetricsDatadogConfig; +import org.apache.hudi.config.metrics.HoodieMetricsGraphiteConfig; +import org.apache.hudi.config.metrics.HoodieMetricsJmxConfig; +import org.apache.hudi.config.metrics.HoodieMetricsPrometheusConfig; import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.keygen.SimpleAvroKeyGenerator; @@ -1459,23 +1464,23 @@ public MetricsReporterType getMetricsReporterType() { } public String getGraphiteServerHost() { - return getString(HoodieMetricsConfig.GRAPHITE_SERVER_HOST_NAME); + return getString(HoodieMetricsGraphiteConfig.GRAPHITE_SERVER_HOST_NAME); } public int getGraphiteServerPort() { - return getInt(HoodieMetricsConfig.GRAPHITE_SERVER_PORT_NUM); + return getInt(HoodieMetricsGraphiteConfig.GRAPHITE_SERVER_PORT_NUM); } public String getGraphiteMetricPrefix() { - return getString(HoodieMetricsConfig.GRAPHITE_METRIC_PREFIX_VALUE); + return getString(HoodieMetricsGraphiteConfig.GRAPHITE_METRIC_PREFIX_VALUE); } public String getJmxHost() { - return getString(HoodieMetricsConfig.JMX_HOST_NAME); + return getString(HoodieMetricsJmxConfig.JMX_HOST_NAME); } public String getJmxPort() { - return getString(HoodieMetricsConfig.JMX_PORT_NUM); + return getString(HoodieMetricsJmxConfig.JMX_PORT_NUM); } public int getDatadogReportPeriodSeconds() { @@ -1777,6 +1782,8 @@ public static class Builder { private boolean isMetadataConfigSet = false; private boolean isLockConfigSet = false; private boolean isPreCommitValidationConfigSet = false; + private boolean isMetricsJmxConfigSet = false; + private boolean isMetricsGraphiteConfigSet = false; public Builder withEngineType(EngineType engineType) { this.engineType = engineType; @@ -1931,6 +1938,18 @@ public Builder withLockConfig(HoodieLockConfig lockConfig) { return this; } + public Builder withMetricsJmxConfig(HoodieMetricsJmxConfig metricsJmxConfig) { + writeConfig.getProps().putAll(metricsJmxConfig.getProps()); + isMetricsJmxConfigSet = true; + return this; + } + + public Builder withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig mericsGraphiteConfig) { + writeConfig.getProps().putAll(mericsGraphiteConfig.getProps()); + isMetricsGraphiteConfigSet = true; + return this; + } + public Builder withPreCommitValidatorConfig(HoodiePreCommitValidatorConfig validatorConfig) { writeConfig.getProps().putAll(validatorConfig.getProps()); isPreCommitValidationConfigSet = true; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java similarity index 62% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java index e1d0b63ef35c4..60369d72fb4fc 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.config; +package org.apache.hudi.config.metrics; import org.apache.hudi.common.config.ConfigClassProperty; import org.apache.hudi.common.config.ConfigGroups; @@ -55,42 +55,6 @@ public class HoodieMetricsConfig extends HoodieConfig { .sinceVersion("0.5.0") .withDocumentation("Type of metrics reporter."); - // Graphite - public static final String GRAPHITE_PREFIX = METRIC_PREFIX + ".graphite"; - - public static final ConfigProperty GRAPHITE_SERVER_HOST_NAME = ConfigProperty - .key(GRAPHITE_PREFIX + ".host") - .defaultValue("localhost") - .sinceVersion("0.5.0") - .withDocumentation("Graphite host to connect to"); - - public static final ConfigProperty GRAPHITE_SERVER_PORT_NUM = ConfigProperty - .key(GRAPHITE_PREFIX + ".port") - .defaultValue(4756) - .sinceVersion("0.5.0") - .withDocumentation("Graphite port to connect to"); - - // Jmx - public static final String JMX_PREFIX = METRIC_PREFIX + ".jmx"; - - public static final ConfigProperty JMX_HOST_NAME = ConfigProperty - .key(JMX_PREFIX + ".host") - .defaultValue("localhost") - .sinceVersion("0.5.1") - .withDocumentation("Jmx host to connect to"); - - public static final ConfigProperty JMX_PORT_NUM = ConfigProperty - .key(JMX_PREFIX + ".port") - .defaultValue(9889) - .sinceVersion("0.5.1") - .withDocumentation("Jmx port to connect to"); - - public static final ConfigProperty GRAPHITE_METRIC_PREFIX_VALUE = ConfigProperty - .key(GRAPHITE_PREFIX + ".metric.prefix") - .noDefaultValue() - .sinceVersion("0.5.1") - .withDocumentation("Standard prefix applied to all metrics. This helps to add datacenter, environment information for e.g"); - // User defined public static final ConfigProperty METRICS_REPORTER_CLASS_NAME = ConfigProperty .key(METRIC_PREFIX + ".reporter.class") @@ -125,51 +89,6 @@ public class HoodieMetricsConfig extends HoodieConfig { */ @Deprecated public static final MetricsReporterType DEFAULT_METRICS_REPORTER_TYPE = METRICS_REPORTER_TYPE_VALUE.defaultValue(); - /** - * @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead - */ - @Deprecated - public static final String GRAPHITE_SERVER_HOST = GRAPHITE_SERVER_HOST_NAME.key(); - /** - * @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead - */ - @Deprecated - public static final String DEFAULT_GRAPHITE_SERVER_HOST = GRAPHITE_SERVER_HOST_NAME.defaultValue(); - /** - * @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead - */ - @Deprecated - public static final String GRAPHITE_SERVER_PORT = GRAPHITE_SERVER_PORT_NUM.key(); - /** - * @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead - */ - @Deprecated - public static final int DEFAULT_GRAPHITE_SERVER_PORT = GRAPHITE_SERVER_PORT_NUM.defaultValue(); - /** - * @deprecated Use {@link #JMX_HOST_NAME} and its methods instead - */ - @Deprecated - public static final String JMX_HOST = JMX_HOST_NAME.key(); - /** - * @deprecated Use {@link #JMX_HOST_NAME} and its methods instead - */ - @Deprecated - public static final String DEFAULT_JMX_HOST = JMX_HOST_NAME.defaultValue(); - /** - * @deprecated Use {@link #JMX_PORT_NUM} and its methods instead - */ - @Deprecated - public static final String JMX_PORT = JMX_PORT_NUM.key(); - /** - * @deprecated Use {@link #JMX_PORT_NUM} and its methods instead - */ - @Deprecated - public static final int DEFAULT_JMX_PORT = JMX_PORT_NUM.defaultValue(); - /** - * @deprecated Use {@link #GRAPHITE_METRIC_PREFIX_VALUE} and its methods instead - */ - @Deprecated - public static final String GRAPHITE_METRIC_PREFIX = GRAPHITE_METRIC_PREFIX_VALUE.key(); /** * @deprecated Use {@link #METRICS_REPORTER_CLASS_NAME} and its methods instead */ @@ -220,31 +139,6 @@ public Builder withReporterType(String reporterType) { return this; } - public Builder toGraphiteHost(String host) { - hoodieMetricsConfig.setValue(GRAPHITE_SERVER_HOST_NAME, host); - return this; - } - - public Builder onGraphitePort(int port) { - hoodieMetricsConfig.setValue(GRAPHITE_SERVER_PORT_NUM, String.valueOf(port)); - return this; - } - - public Builder toJmxHost(String host) { - hoodieMetricsConfig.setValue(JMX_HOST_NAME, host); - return this; - } - - public Builder onJmxPort(String port) { - hoodieMetricsConfig.setValue(JMX_PORT_NUM, port); - return this; - } - - public Builder usePrefix(String prefix) { - hoodieMetricsConfig.setValue(GRAPHITE_METRIC_PREFIX_VALUE, prefix); - return this; - } - public Builder withReporterClass(String className) { hoodieMetricsConfig.setValue(METRICS_REPORTER_CLASS_NAME, className); return this; @@ -267,6 +161,10 @@ public HoodieMetricsConfig build() { HoodieMetricsPrometheusConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build()); hoodieMetricsConfig.setDefaultOnCondition(reporterType == MetricsReporterType.PROMETHEUS, HoodieMetricsPrometheusConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build()); + hoodieMetricsConfig.setDefaultOnCondition(reporterType == MetricsReporterType.JMX, + HoodieMetricsJmxConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build()); + hoodieMetricsConfig.setDefaultOnCondition(reporterType == MetricsReporterType.GRAPHITE, + HoodieMetricsGraphiteConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build()); return hoodieMetricsConfig; } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsDatadogConfig.java similarity index 98% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsDatadogConfig.java index d63cb0fefa309..3fc306b8cdd3b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsDatadogConfig.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.config; +package org.apache.hudi.config.metrics; import org.apache.hudi.common.config.ConfigClassProperty; import org.apache.hudi.common.config.ConfigGroups; @@ -27,7 +27,7 @@ import java.util.Properties; -import static org.apache.hudi.config.HoodieMetricsConfig.METRIC_PREFIX; +import static org.apache.hudi.config.metrics.HoodieMetricsConfig.METRIC_PREFIX; /** * Configs for Datadog reporter type. diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsGraphiteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsGraphiteConfig.java new file mode 100644 index 0000000000000..12987a7e1574e --- /dev/null +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsGraphiteConfig.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.config.metrics; + +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; +import org.apache.hudi.common.config.ConfigProperty; +import org.apache.hudi.common.config.HoodieConfig; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.Properties; + +import static org.apache.hudi.config.metrics.HoodieMetricsConfig.METRIC_PREFIX; + +/** + * Configs for Graphite reporter type. + *

+ * {@link org.apache.hudi.metrics.MetricsReporterType#GRAPHITE} + */ +@ConfigClassProperty(name = "Metrics Configurations for Graphite", + groupName = ConfigGroups.Names.METRICS, + description = "Enables reporting on Hudi metrics using Graphite. " + + " Hudi publishes metrics on every commit, clean, rollback etc.") +public class HoodieMetricsGraphiteConfig extends HoodieConfig { + + public static final String GRAPHITE_PREFIX = METRIC_PREFIX + ".graphite"; + + public static final ConfigProperty GRAPHITE_SERVER_HOST_NAME = ConfigProperty + .key(GRAPHITE_PREFIX + ".host") + .defaultValue("localhost") + .sinceVersion("0.5.0") + .withDocumentation("Graphite host to connect to."); + + public static final ConfigProperty GRAPHITE_SERVER_PORT_NUM = ConfigProperty + .key(GRAPHITE_PREFIX + ".port") + .defaultValue(4756) + .sinceVersion("0.5.0") + .withDocumentation("Graphite port to connect to."); + + public static final ConfigProperty GRAPHITE_METRIC_PREFIX_VALUE = ConfigProperty + .key(GRAPHITE_PREFIX + ".metric.prefix") + .noDefaultValue() + .sinceVersion("0.5.1") + .withDocumentation("Standard prefix applied to all metrics. This helps to add datacenter, environment information for e.g"); + + /** + * @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead + */ + @Deprecated + public static final String GRAPHITE_SERVER_HOST = GRAPHITE_SERVER_HOST_NAME.key(); + /** + * @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_GRAPHITE_SERVER_HOST = GRAPHITE_SERVER_HOST_NAME.defaultValue(); + /** + * @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead + */ + @Deprecated + public static final String GRAPHITE_SERVER_PORT = GRAPHITE_SERVER_PORT_NUM.key(); + /** + * @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead + */ + @Deprecated + public static final int DEFAULT_GRAPHITE_SERVER_PORT = GRAPHITE_SERVER_PORT_NUM.defaultValue(); + /** + * @deprecated Use {@link #GRAPHITE_METRIC_PREFIX_VALUE} and its methods instead + */ + @Deprecated + public static final String GRAPHITE_METRIC_PREFIX = GRAPHITE_METRIC_PREFIX_VALUE.key(); + + private HoodieMetricsGraphiteConfig() { + super(); + } + + public static HoodieMetricsGraphiteConfig.Builder newBuilder() { + return new HoodieMetricsGraphiteConfig.Builder(); + } + + public static class Builder { + + private final HoodieMetricsGraphiteConfig hoodieMetricsGraphiteConfig = new HoodieMetricsGraphiteConfig(); + + public HoodieMetricsGraphiteConfig.Builder fromFile(File propertiesFile) throws IOException { + try (FileReader reader = new FileReader(propertiesFile)) { + this.hoodieMetricsGraphiteConfig.getProps().load(reader); + return this; + } + } + + public HoodieMetricsGraphiteConfig.Builder fromProperties(Properties props) { + this.hoodieMetricsGraphiteConfig.getProps().putAll(props); + return this; + } + + public HoodieMetricsGraphiteConfig.Builder toGraphiteHost(String host) { + hoodieMetricsGraphiteConfig.setValue(GRAPHITE_SERVER_HOST_NAME, host); + return this; + } + + public HoodieMetricsGraphiteConfig.Builder onGraphitePort(int port) { + hoodieMetricsGraphiteConfig.setValue(GRAPHITE_SERVER_PORT_NUM, String.valueOf(port)); + return this; + } + + public HoodieMetricsGraphiteConfig.Builder usePrefix(String prefix) { + hoodieMetricsGraphiteConfig.setValue(GRAPHITE_METRIC_PREFIX_VALUE, prefix); + return this; + } + + public HoodieMetricsGraphiteConfig build() { + hoodieMetricsGraphiteConfig.setDefaults(HoodieMetricsGraphiteConfig.class.getName()); + return hoodieMetricsGraphiteConfig; + } + } +} diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsJmxConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsJmxConfig.java new file mode 100644 index 0000000000000..e3a57a1c5caf4 --- /dev/null +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsJmxConfig.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.config.metrics; + +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; +import org.apache.hudi.common.config.ConfigProperty; +import org.apache.hudi.common.config.HoodieConfig; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.Properties; + +import static org.apache.hudi.config.metrics.HoodieMetricsConfig.METRIC_PREFIX; + +/** + * Configs for Jmx reporter type. + *

+ * {@link org.apache.hudi.metrics.MetricsReporterType#JMX} + */ +@ConfigClassProperty(name = "Metrics Configurations for Jmx", + groupName = ConfigGroups.Names.METRICS, + description = "Enables reporting on Hudi metrics using Jmx. " + + " Hudi publishes metrics on every commit, clean, rollback etc.") +public class HoodieMetricsJmxConfig extends HoodieConfig { + + public static final String JMX_PREFIX = METRIC_PREFIX + ".jmx"; + + public static final ConfigProperty JMX_HOST_NAME = ConfigProperty + .key(JMX_PREFIX + ".host") + .defaultValue("localhost") + .sinceVersion("0.5.1") + .withDocumentation("Jmx host to connect to"); + + public static final ConfigProperty JMX_PORT_NUM = ConfigProperty + .key(JMX_PREFIX + ".port") + .defaultValue(9889) + .sinceVersion("0.5.1") + .withDocumentation("Jmx port to connect to"); + + /** + * @deprecated Use {@link #JMX_HOST_NAME} and its methods instead + */ + @Deprecated + public static final String JMX_HOST = JMX_HOST_NAME.key(); + /** + * @deprecated Use {@link #JMX_HOST_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_JMX_HOST = JMX_HOST_NAME.defaultValue(); + /** + * @deprecated Use {@link #JMX_PORT_NUM} and its methods instead + */ + @Deprecated + public static final String JMX_PORT = JMX_PORT_NUM.key(); + /** + * @deprecated Use {@link #JMX_PORT_NUM} and its methods instead + */ + @Deprecated + public static final int DEFAULT_JMX_PORT = JMX_PORT_NUM.defaultValue(); + + private HoodieMetricsJmxConfig() { + super(); + } + + public static HoodieMetricsJmxConfig.Builder newBuilder() { + return new HoodieMetricsJmxConfig.Builder(); + } + + public static class Builder { + + private final HoodieMetricsJmxConfig hoodieMetricsJmxConfig = new HoodieMetricsJmxConfig(); + + public HoodieMetricsJmxConfig.Builder fromFile(File propertiesFile) throws IOException { + try (FileReader reader = new FileReader(propertiesFile)) { + this.hoodieMetricsJmxConfig.getProps().load(reader); + return this; + } + } + + public HoodieMetricsJmxConfig.Builder fromProperties(Properties props) { + this.hoodieMetricsJmxConfig.getProps().putAll(props); + return this; + } + + public HoodieMetricsJmxConfig.Builder toJmxHost(String host) { + hoodieMetricsJmxConfig.setValue(JMX_HOST_NAME, host); + return this; + } + + public HoodieMetricsJmxConfig.Builder onJmxPort(String port) { + hoodieMetricsJmxConfig.setValue(JMX_PORT_NUM, port); + return this; + } + + public HoodieMetricsJmxConfig build() { + hoodieMetricsJmxConfig.setDefaults(HoodieMetricsJmxConfig.class.getName()); + return hoodieMetricsJmxConfig; + } + } +} diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsPrometheusConfig.java similarity index 78% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsPrometheusConfig.java index 9ec2ec62e460e..c04e8aa1e980e 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsPrometheusConfig.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.config; +package org.apache.hudi.config.metrics; import org.apache.hudi.common.config.ConfigClassProperty; import org.apache.hudi.common.config.ConfigGroups; @@ -25,8 +25,14 @@ import java.util.Properties; -import static org.apache.hudi.config.HoodieMetricsConfig.METRIC_PREFIX; +import static org.apache.hudi.config.metrics.HoodieMetricsConfig.METRIC_PREFIX; +/** + * Configs for Prometheus/Pushgaeway reporter type. + *

+ * {@link org.apache.hudi.metrics.MetricsReporterType#PROMETHEUS} + * {@link org.apache.hudi.metrics.MetricsReporterType#PROMETHEUS_PUSHGATEWAY} + */ @ConfigClassProperty(name = "Metrics Configurations for Prometheus", groupName = ConfigGroups.Names.METRICS, description = "Enables reporting on Hudi metrics using Prometheus. " @@ -174,6 +180,41 @@ public Builder fromProperties(Properties props) { return this; } + public HoodieMetricsPrometheusConfig.Builder withPushgatewayHostName(String hostName) { + hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_HOST_NAME, String.valueOf(hostName)); + return this; + } + + public HoodieMetricsPrometheusConfig.Builder withPushgatewayPortNum(Integer pushgatewayPortNum) { + hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_PORT_NUM, String.valueOf(pushgatewayPortNum)); + return this; + } + + public HoodieMetricsPrometheusConfig.Builder withPushgatewayReportPeriodInSeconds(String periodTime) { + hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS, periodTime); + return this; + } + + public HoodieMetricsPrometheusConfig.Builder withPushgatewayDeleteOnShutdownEnable(boolean deleteOnShutdownEnable) { + hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE, String.valueOf(deleteOnShutdownEnable)); + return this; + } + + public HoodieMetricsPrometheusConfig.Builder withPushgatewayJobname(String jobname) { + hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_JOBNAME, jobname); + return this; + } + + public HoodieMetricsPrometheusConfig.Builder withPushgatewayRandomJobnameSuffix(boolean randomJobnameSuffix) { + hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX, String.valueOf(randomJobnameSuffix)); + return this; + } + + public HoodieMetricsPrometheusConfig.Builder withPrometheusPortNum(int prometheusPortNum) { + hoodieMetricsPrometheusConfig.setValue(PROMETHEUS_PORT_NUM, String.valueOf(prometheusPortNum)); + return this; + } + public HoodieMetricsPrometheusConfig build() { hoodieMetricsPrometheusConfig.setDefaults(HoodieMetricsPrometheusConfig.class.getName()); return hoodieMetricsPrometheusConfig; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index ab5fa994deb76..f7979459458cd 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -46,8 +46,10 @@ import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieCompactionConfig; -import org.apache.hudi.config.HoodieMetricsConfig; +import org.apache.hudi.config.metrics.HoodieMetricsConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.config.metrics.HoodieMetricsGraphiteConfig; +import org.apache.hudi.config.metrics.HoodieMetricsJmxConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieMetadataException; @@ -175,19 +177,22 @@ private HoodieWriteConfig createMetadataWriteConfig(HoodieWriteConfig writeConfi .withFinalizeWriteParallelism(parallelism); if (writeConfig.isMetricsOn()) { - HoodieMetricsConfig.Builder metricsConfig = HoodieMetricsConfig.newBuilder() + builder.withMetricsConfig(HoodieMetricsConfig.newBuilder() .withReporterType(writeConfig.getMetricsReporterType().toString()) .withExecutorMetrics(writeConfig.isExecutorMetricsEnabled()) - .on(true); + .on(true).build()); switch (writeConfig.getMetricsReporterType()) { case GRAPHITE: - metricsConfig.onGraphitePort(writeConfig.getGraphiteServerPort()) + builder.withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig.newBuilder() + .onGraphitePort(writeConfig.getGraphiteServerPort()) .toGraphiteHost(writeConfig.getGraphiteServerHost()) - .usePrefix(writeConfig.getGraphiteMetricPrefix()); + .usePrefix(writeConfig.getGraphiteMetricPrefix()).build()); break; case JMX: - metricsConfig.onJmxPort(writeConfig.getJmxPort()) - .toJmxHost(writeConfig.getJmxHost()); + builder.withMetricsJmxConfig(HoodieMetricsJmxConfig.newBuilder() + .onJmxPort(writeConfig.getJmxPort()) + .toJmxHost(writeConfig.getJmxHost()) + .build()); break; case DATADOG: case PROMETHEUS: @@ -198,10 +203,7 @@ private HoodieWriteConfig createMetadataWriteConfig(HoodieWriteConfig writeConfi default: throw new HoodieMetadataException("Unsupported Metrics Reporter type " + writeConfig.getMetricsReporterType()); } - - builder.withMetricsConfig(metricsConfig.build()); } - return builder.build(); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java index 016e070d7fbce..aa486e9b95245 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java @@ -19,7 +19,7 @@ package org.apache.hudi.metrics.datadog; -import org.apache.hudi.config.HoodieMetricsDatadogConfig; +import org.apache.hudi.config.metrics.HoodieMetricsDatadogConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.junit.jupiter.api.Test; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java index bec7ee4a8b3a2..b091359b2dda5 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java @@ -47,9 +47,10 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; -import org.apache.hudi.config.HoodieMetricsConfig; +import org.apache.hudi.config.metrics.HoodieMetricsConfig; import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.config.metrics.HoodieMetricsGraphiteConfig; import org.apache.hudi.exception.HoodieMetadataException; import org.apache.hudi.exception.TableNotFoundException; import org.apache.hudi.index.HoodieIndex; @@ -1305,7 +1306,9 @@ private HoodieWriteConfig.Builder getWriteConfigBuilder(HoodieFailedWritesCleani .enable(useFileListingMetadata) .enableMetrics(enableMetrics).build()) .withMetricsConfig(HoodieMetricsConfig.newBuilder().on(enableMetrics) - .withExecutorMetrics(true).usePrefix("unit-test").build()); + .withExecutorMetrics(true).build()) + .withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig.newBuilder() + .usePrefix("unit-test").build()); } @Override