diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java index de413283a85..57f44568a6f 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java @@ -1195,17 +1195,7 @@ private Collection> findMetricFunctions(Multimap> findMetricFunctions(Multimap hostAggregateMap, String phoenixTableName) throws SQLException { diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java index 8f18aa9329f..d9bf4976ddb 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java @@ -296,6 +296,9 @@ public class TimelineMetricConfiguration { public static final String TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS = "timeline.metrics.support.multiple.clusters"; + public static final String TIMELINE_METRICS_EVENT_METRIC_PATTERNS = + "timeline.metrics.downsampler.event.metric.patterns"; + public static final String HOST_APP_ID = "HOST"; public static final String DEFAULT_INSTANCE_PORT = "12001"; diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java index 62f8137d178..af978af4c3c 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java @@ -429,11 +429,11 @@ private void runDownSamplerQuery(Connection conn, Condition condition) { try { stmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition); - LOG.debug("Downsampler Query issued..."); + LOG.debug("Downsampler Query issued : " + condition.getStatement()); if (condition.doUpdate()) { int rows = stmt.executeUpdate(); conn.commit(); - LOG.info(rows + " row(s) updated in downsampling."); + LOG.debug(rows + " row(s) updated in downsampling."); } else { rs = stmt.executeQuery(); } diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerUtils.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerUtils.java index b320e47212d..649eceefed1 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerUtils.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerUtils.java @@ -24,7 +24,6 @@ import org.apache.hadoop.conf.Configuration; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; @@ -38,7 +37,8 @@ public class DownSamplerUtils { public static final String downSamplerConfigPrefix = "timeline.metrics.downsampler."; public static final String downSamplerMetricPatternsConfig = "metric.patterns"; - public static final String topNDownSampler = "topn"; + public static final String topNDownSamplerKey = "topn"; + public static final String eventDownSamplerKey = "event"; private static final Log LOG = LogFactory.getLog(DownSamplerUtils.class); @@ -108,10 +108,14 @@ public static CustomDownSampler getDownSamplerByType(String type, Map conf) { + String metricPatterns = conf.get(DownSamplerUtils.downSamplerConfigPrefix + DownSamplerUtils.eventDownSamplerKey + "." + + DownSamplerUtils.downSamplerMetricPatternsConfig); + + return new EventMetricDownSampler(metricPatterns); + } + + public EventMetricDownSampler(String metricPatterns) { + this.metricPatterns = metricPatterns; + } + + @Override + public boolean validateConfigs() { + return true; + } + + @Override + public List prepareDownSamplingStatement(Long startTime, Long endTime, String tableName) { + List stmts = new ArrayList<>(); + List metricPatternList = Arrays.asList(metricPatterns.split(",")); + + String aggregateColumnName = "METRIC_COUNT"; + + if (tableName.equals(METRICS_CLUSTER_AGGREGATE_TABLE_NAME)) { + aggregateColumnName = "HOSTS_COUNT"; + } + + for (String metricPattern : metricPatternList) { + String metricPatternClause = "'" + metricPattern + "'"; + if (tableName.contains("RECORD")) { + stmts.add(String.format(EVENT_DOWNSAMPLER_HOST_METRIC_SELECT_SQL, + endTime, tableName, metricPatternClause, + startTime, endTime)); + } else { + stmts.add(String.format(EVENT_DOWNSAMPLER_CLUSTER_METRIC_SELECT_SQL, + endTime, aggregateColumnName, tableName, metricPatternClause, + startTime, endTime)); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Downsampling Stmt: " + stmts.toString()); + } + return stmts; + } +} \ No newline at end of file diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java index e01402f40db..50eb2907b2e 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.SERVER_SIDE_TIMESIFT_ADJUSTMENT; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_EVENT_METRIC_PATTERNS; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATION_SQL_FILTERS; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_METRIC_SQL; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME; @@ -31,8 +32,12 @@ import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; @@ -63,7 +68,8 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre private final boolean interpolationEnabled; private TimelineMetricMetadataManager metadataManagerInstance; private String skipAggrPatternStrings; - + private String skipInterpolationMetricPatternStrings; + private Set skipInterpolationMetricPatterns = new HashSet<>(); public TimelineMetricClusterAggregatorSecond(AggregationTaskRunner.AGGREGATOR_NAME aggregatorName, TimelineMetricMetadataManager metadataManager, @@ -88,6 +94,15 @@ public TimelineMetricClusterAggregatorSecond(AggregationTaskRunner.AGGREGATOR_NA this.serverTimeShiftAdjustment = Long.parseLong(metricsConf.get(SERVER_SIDE_TIMESIFT_ADJUSTMENT, "90000")); this.interpolationEnabled = Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED, "true")); this.skipAggrPatternStrings = metricsConf.get(TIMELINE_METRIC_AGGREGATION_SQL_FILTERS); + this.skipInterpolationMetricPatternStrings = metricsConf.get(TIMELINE_METRICS_EVENT_METRIC_PATTERNS, ""); + + if (StringUtils.isNotEmpty(skipInterpolationMetricPatternStrings)) { + for (String patternString : skipInterpolationMetricPatternStrings.split(",")) { + String javaPatternString = hBaseAccessor.getJavaRegexFromSqlRegex(patternString); + LOG.info("SQL pattern " + patternString + " converted to Java pattern : " + javaPatternString); + skipInterpolationMetricPatterns.add(Pattern.compile(javaPatternString)); + } + } } @Override @@ -326,6 +341,14 @@ private void interpolateMissingPeriods(Map timeli Map timeSliceValueMap) { + for (Pattern pattern : skipInterpolationMetricPatterns) { + Matcher m = pattern.matcher(timelineMetric.getMetricName()); + if (m.matches()) { + LOG.debug("Skipping interpolation for " + timelineMetric.getMetricName()); + return; + } + } + if (StringUtils.isNotEmpty(timelineMetric.getType()) && "COUNTER".equalsIgnoreCase(timelineMetric.getType())) { //For Counter Based metrics, ok to do interpolation and extrapolation diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java index b5f49fb07c8..9606ee31965 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java @@ -72,7 +72,7 @@ public SingleValuedTimelineMetric getAggregatedTimelineMetricFromResultSet(Resul value = rs.getDouble("METRIC_MAX"); break; case SUM: - value = rs.getDouble("METRIC_SUM") / rs.getInt("METRIC_COUNT"); + value = rs.getDouble("METRIC_SUM"); break; default: value = rs.getDouble("METRIC_SUM") / rs.getInt("METRIC_COUNT"); diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TopNDownSampler.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TopNDownSampler.java index 59e44530f6c..520da0a8cc0 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TopNDownSampler.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TopNDownSampler.java @@ -39,7 +39,7 @@ public class TopNDownSampler implements CustomDownSampler { protected String metricPatterns; public static TopNDownSampler fromConfig(Map conf) { - String metricPatterns = conf.get(DownSamplerUtils.downSamplerConfigPrefix + "topn." + + String metricPatterns = conf.get(DownSamplerUtils.downSamplerConfigPrefix + DownSamplerUtils.topNDownSamplerKey + "." + DownSamplerUtils.downSamplerMetricPatternsConfig); String topNString = conf.get(DownSamplerUtils.downSamplerConfigPrefix + "topn.value"); diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java index d39230d3791..e55ff61324b 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java @@ -329,7 +329,7 @@ public class PhoenixTransactSQL { "INTO %s (METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, SERVER_TIME, UNITS, " + "METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) " + "SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, %s AS SERVER_TIME, UNITS, " + - "SUM(METRIC_SUM), SUM(METRIC_COUNT), MAX(METRIC_MAX), MIN(METRIC_MIN) " + + "ROUND(SUM(METRIC_SUM)/SUM(METRIC_COUNT),2), SUM(METRIC_COUNT), MAX(METRIC_MAX), MIN(METRIC_MIN) " + "FROM %s WHERE%s SERVER_TIME > %s AND SERVER_TIME <= %s " + "GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS"; @@ -364,6 +364,18 @@ public class PhoenixTransactSQL { " %s AS SERVER_TIME, UNITS, %s, 1, %s, %s FROM %s WHERE METRIC_NAME LIKE %s AND SERVER_TIME > %s AND SERVER_TIME <= %s " + "GROUP BY METRIC_NAME, APP_ID, INSTANCE_ID, UNITS ORDER BY %s DESC LIMIT %s"; + /** + * Event based downsampler SELECT query. + */ + public static final String EVENT_DOWNSAMPLER_HOST_METRIC_SELECT_SQL = "SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, " + + "%s AS SERVER_TIME, UNITS, SUM(METRIC_SUM), SUM(METRIC_COUNT), MAX(METRIC_MAX), MIN(METRIC_MIN) " + + "FROM %s WHERE METRIC_NAME LIKE %s AND SERVER_TIME > %s AND SERVER_TIME <= %s GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS"; + + public static final String EVENT_DOWNSAMPLER_CLUSTER_METRIC_SELECT_SQL = "SELECT METRIC_NAME, APP_ID, " + + "INSTANCE_ID, %s AS SERVER_TIME, UNITS, SUM(METRIC_SUM), SUM(%s), " + + "MAX(METRIC_MAX), MIN(METRIC_MIN) FROM %s WHERE METRIC_NAME LIKE %s AND SERVER_TIME > %s AND " + + "SERVER_TIME <= %s GROUP BY METRIC_NAME, APP_ID, INSTANCE_ID, UNITS"; + public static final String METRICS_RECORD_TABLE_NAME = "METRIC_RECORD"; public static final String CONTAINER_METRICS_TABLE_NAME = "CONTAINER_METRICS"; diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerTest.java index a0fea40d7b4..d02d2a89af2 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerTest.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerTest.java @@ -92,4 +92,24 @@ public void testPrepareTopNDownSamplingStatement() throws Exception { "METRIC_NAME LIKE 'pattern1' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " + "GROUP BY METRIC_NAME, APP_ID, INSTANCE_ID, UNITS ORDER BY SUM(METRIC_SUM) DESC LIMIT 4")); } + + @Test + public void testPrepareEventDownSamplingStatement() throws Exception { + Configuration configuration = new Configuration(); + configuration.setIfUnset("timeline.metrics.downsampler.event.metric.patterns", "pattern1,pattern2"); + + Map conf = configuration.getValByRegex(DownSamplerUtils.downSamplerConfigPrefix); + + EventMetricDownSampler eventMetricDownSampler = EventMetricDownSampler.fromConfig(conf); + List stmts = eventMetricDownSampler.prepareDownSamplingStatement(14000000l, 14100000l, "METRIC_RECORD"); + Assert.assertEquals(stmts.size(),2); + + Assert.assertTrue(stmts.get(0).equals("SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, " + + "UNITS, SUM(METRIC_SUM), SUM(METRIC_COUNT), MAX(METRIC_MAX), MIN(METRIC_MIN) FROM METRIC_RECORD WHERE METRIC_NAME " + + "LIKE 'pattern1' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS")); + + Assert.assertTrue(stmts.get(1).equals("SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, " + + "UNITS, SUM(METRIC_SUM), SUM(METRIC_COUNT), MAX(METRIC_MAX), MIN(METRIC_MIN) FROM METRIC_RECORD WHERE METRIC_NAME " + + "LIKE 'pattern2' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS")); + } } diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml index b9f534e66ac..bcbcabd1a1b 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml @@ -720,4 +720,16 @@ {{cluster_zookeeper_clientPort}} + + timeline.metrics.downsampler.event.metric.patterns + + + Commas separated list of metric name regular expressions that are like events. No interpolation will be done for such + metrics, and the downsampling SUM aggregators will sum the values across time instead of averaging them out. + + + true + + +