Zero filling missing bucket in Explore Data

projects/observability/src/shared/dashboard/data/graphql/explore/explore-cartesian-data-source.model.test.ts

@@ -1,9 +1,8 @@
-import { ColorService, TimeDuration, TimeUnit } from '@hypertrace/common';
+import { ColorService, FixedTimeRange, TimeDuration, TimeUnit } from '@hypertrace/common';
 import { createModelFactory } from '@hypertrace/dashboards/testing';
 import {
   AttributeMetadataType,
   GraphQlQueryEventService,
-  GraphQlTimeRange,
   MetadataService,
   MetricAggregationType
 } from '@hypertrace/distributed-tracing';
@@ -29,6 +28,8 @@ import { ExploreCartesianDataSourceModel, ExplorerData } from './explore-cartesi
 
 describe('Explore cartesian data source model', () => {
   const testInterval = new TimeDuration(5, TimeUnit.Minute);
+  const endTime = new Date('2021-05-11T00:35:00.000Z');
+  const startTime = new Date(endTime.getTime() - 2 * testInterval.toMillis());
 
   const modelFactory = createModelFactory({
     providers: [
@@ -74,7 +75,7 @@ describe('Explore cartesian data source model', () => {
   beforeEach(() => {
     model = modelFactory(TestExploreCartesianDataSourceModel, {
       api: {
-        getTimeRange: jest.fn().mockReturnValue(new GraphQlTimeRange(2, 3))
+        getTimeRange: () => new FixedTimeRange(startTime, endTime)
       }
     }).model;
   });
@@ -100,14 +101,14 @@ describe('Explore cartesian data source model', () => {
                   value: 10,
                   type: AttributeMetadataType.Number
                 },
-                [GQL_EXPLORE_RESULT_INTERVAL_KEY]: new Date(0)
+                [GQL_EXPLORE_RESULT_INTERVAL_KEY]: startTime
               },
               {
                 'sum(foo)': {
                   value: 15,
                   type: AttributeMetadataType.Number
                 },
-                [GQL_EXPLORE_RESULT_INTERVAL_KEY]: new Date(1)
+                [GQL_EXPLORE_RESULT_INTERVAL_KEY]: endTime
               }
             ]
           },
@@ -122,11 +123,15 @@ describe('Explore cartesian data source model', () => {
               type: CartesianSeriesVisualizationType.Line,
               data: [
                 {
-                  timestamp: new Date(0),
+                  timestamp: startTime,
                   value: 10
                 },
                 {
-                  timestamp: new Date(1),
+                  timestamp: new Date('2021-05-11T00:30:00.000Z'),
+                  value: 0
+                },
+                {
+                  timestamp: endTime,
                   value: 15
                 }
               ]
@@ -231,7 +236,7 @@ describe('Explore cartesian data source model', () => {
                   value: 'first',
                   type: AttributeMetadataType.String
                 },
-                [GQL_EXPLORE_RESULT_INTERVAL_KEY]: new Date(0)
+                [GQL_EXPLORE_RESULT_INTERVAL_KEY]: startTime
               },
               {
                 'sum(foo)': {
@@ -242,7 +247,7 @@ describe('Explore cartesian data source model', () => {
                   value: 'first',
                   type: AttributeMetadataType.String
                 },
-                [GQL_EXPLORE_RESULT_INTERVAL_KEY]: new Date(1)
+                [GQL_EXPLORE_RESULT_INTERVAL_KEY]: endTime
               },
               {
                 'sum(foo)': {
@@ -253,7 +258,7 @@ describe('Explore cartesian data source model', () => {
                   value: 'second',
                   type: AttributeMetadataType.String
                 },
-                [GQL_EXPLORE_RESULT_INTERVAL_KEY]: new Date(0)
+                [GQL_EXPLORE_RESULT_INTERVAL_KEY]: startTime
               },
               {
                 'sum(foo)': {
@@ -264,7 +269,7 @@ describe('Explore cartesian data source model', () => {
                   value: 'second',
                   type: AttributeMetadataType.String
                 },
-                [GQL_EXPLORE_RESULT_INTERVAL_KEY]: new Date(1)
+                [GQL_EXPLORE_RESULT_INTERVAL_KEY]: endTime
               }
             ]
           },
@@ -279,11 +284,15 @@ describe('Explore cartesian data source model', () => {
               type: CartesianSeriesVisualizationType.Area,
               data: [
                 {
-                  timestamp: new Date(0),
+                  timestamp: startTime,
                   value: 10
                 },
                 {
-                  timestamp: new Date(1),
+                  timestamp: new Date('2021-05-11T00:30:00.000Z'),
+                  value: 0
+                },
+                {
+                  timestamp: endTime,
                   value: 15
                 }
               ]
@@ -294,11 +303,15 @@ describe('Explore cartesian data source model', () => {
               type: CartesianSeriesVisualizationType.Area,
               data: [
                 {
-                  timestamp: new Date(0),
+                  timestamp: startTime,
                   value: 20
                 },
                 {
-                  timestamp: new Date(1),
+                  timestamp: new Date('2021-05-11T00:30:00.000Z'),
+                  value: 0
+                },
+                {
+                  timestamp: endTime,
                   value: 25
                 }
               ]

projects/observability/src/shared/dashboard/data/graphql/explore/explore-cartesian-data-source.model.ts

@@ -1,5 +1,10 @@
 import { ColorService, forkJoinSafeEmpty, RequireBy, TimeDuration } from '@hypertrace/common';
-import { GraphQlDataSourceModel, GraphQlFilter, MetadataService } from '@hypertrace/distributed-tracing';
+import {
+  GraphQlDataSourceModel,
+  GraphQlFilter,
+  GraphQlTimeRange,
+  MetadataService
+} from '@hypertrace/distributed-tracing';
 import { ModelInject } from '@hypertrace/hyperdash-angular';
 import { isEmpty } from 'lodash-es';
 import { NEVER, Observable, of } from 'rxjs';
@@ -36,43 +41,58 @@ export abstract class ExploreCartesianDataSourceModel extends GraphQlDataSourceM
 
   protected fetchResults(interval: TimeDuration | 'AUTO'): Observable<CartesianResult<ExplorerData>> {
     const requestState = this.buildRequestState(interval);
-
+    const timeRange = this.getTimeRangeOrThrow();
     if (requestState === undefined) {
       return NEVER;
     }
 
     return this.query<ExploreGraphQlQueryHandlerService>(inheritedFilters =>
-      this.buildExploreRequest(requestState, this.getFilters(inheritedFilters))
-    ).pipe(mergeMap(response => this.mapResponseData(requestState, response)));
+      this.buildExploreRequest(requestState, this.getFilters(inheritedFilters), timeRange)
+    ).pipe(
+      mergeMap(response =>
+        this.mapResponseData(requestState, response, requestState.interval as TimeDuration, timeRange)
+      )
+    );
   }
 
   protected mapResponseData(
     requestState: ExploreRequestState,
-    response: GraphQlExploreResponse
+    response: GraphQlExploreResponse,
+    interval: TimeDuration,
+    timeRange: GraphQlTimeRange
   ): Observable<CartesianResult<ExplorerData>> {
-    return this.getAllData(requestState, response).pipe(
+    return this.getAllData(requestState, response, interval, timeRange).pipe(
       map(explorerResults => ({
         series: explorerResults,
         bands: []
       }))
     );
   }
 
-  protected buildExploreRequest(requestState: ExploreRequestState, filters: GraphQlFilter[]): GraphQlExploreRequest {
+  protected buildExploreRequest(
+    requestState: ExploreRequestState,
+    filters: GraphQlFilter[],
+    timeRange: GraphQlTimeRange
+  ): GraphQlExploreRequest {
     return {
       requestType: EXPLORE_GQL_REQUEST,
       selections: requestState.series.map(series => series.specification),
       context: requestState.context,
       limit: requestState.resultLimit,
-      timeRange: this.getTimeRangeOrThrow(),
+      timeRange: timeRange,
       interval: requestState.interval as TimeDuration,
       filters: filters,
       groupBy: requestState.groupBy
     };
   }
 
-  private getAllData(request: ExploreRequestState, response: GraphQlExploreResponse): Observable<ExplorerSeries[]> {
-    return this.buildAllSeries(request, new ExploreResult(response));
+  private getAllData(
+    request: ExploreRequestState,
+    response: GraphQlExploreResponse,
+    interval?: TimeDuration,
+    timeRange?: GraphQlTimeRange
+  ): Observable<ExplorerSeries[]> {
+    return this.buildAllSeries(request, new ExploreResult(response, interval, timeRange));
   }
 
   protected buildAllSeries(request: ExploreRequestState, result: ExploreResult): Observable<ExplorerSeries[]> {

projects/observability/src/shared/dashboard/data/graphql/explore/explore-result.ts

@@ -1,4 +1,5 @@
-import { MetricAggregationType } from '@hypertrace/distributed-tracing';
+import { TimeDuration } from '@hypertrace/common';
+import { GraphQlTimeRange, MetricAggregationType } from '@hypertrace/distributed-tracing';
 import { groupBy } from 'lodash-es';
 import { MetricTimeseriesInterval } from '../../../../graphql/model/metric/metric-timeseries';
 import { ExploreSpecification } from '../../../../graphql/model/schema/specifications/explore-specification';
@@ -15,7 +16,11 @@ export class ExploreResult {
 
   private readonly specBuilder: ExploreSpecificationBuilder = new ExploreSpecificationBuilder();
 
-  public constructor(private readonly response: GraphQlExploreResponse) {}
+  public constructor(
+    private readonly response: GraphQlExploreResponse,
+    private readonly interval?: TimeDuration,
+    private readonly timeRange?: GraphQlTimeRange
+  ) {}
 
   public getTimeSeriesData(metricKey: string, aggregation: MetricAggregationType): MetricTimeseriesInterval[] {
     return this.extractTimeseriesForSpec(this.specBuilder.exploreSpecificationForKey(metricKey, aggregation));
@@ -42,7 +47,7 @@ export class ExploreResult {
     return new Map(
       Object.entries(groupedResults).map(([concatenatedGroupNames, results]) => [
         concatenatedGroupNames.split(','),
-        results.map(result => this.resultToTimeseriesInterval(result, spec))
+        this.resultsToTimeseriesIntervals(results, spec)
       ])
     );
   }
@@ -52,7 +57,7 @@ export class ExploreResult {
   }
 
   private extractTimeseriesForSpec(spec: ExploreSpecification): MetricTimeseriesInterval[] {
-    return this.resultsContainingSpec(spec).map(result => this.resultToTimeseriesInterval(result, spec));
+    return this.resultsToTimeseriesIntervals(this.resultsContainingSpec(spec), spec);
   }
 
   private resultToGroupData(
@@ -72,6 +77,43 @@ export class ExploreResult {
       .map(name => (name === ExploreResult.OTHER_SERVER_GROUP_NAME ? ExploreResult.OTHER_UI_GROUP_NAME : name));
   }
 
+  private resultsToTimeseriesIntervals(
+    results: GraphQlExploreResult[],
+    spec: ExploreSpecification
+  ): MetricTimeseriesInterval[] {
+    if (this.interval !== undefined && this.timeRange !== undefined) {
+      // This should add missing data to array
+
+      // Add all intervals
+      const buckets = [];
+      const intervalDuration = this.interval.toMillis();
+      const startTime = Math.floor(this.timeRange.from.valueOf() / intervalDuration) * intervalDuration;
+      const endTime = Math.ceil(this.timeRange.to.valueOf() / intervalDuration) * intervalDuration;
+
+      for (let timestamp = startTime; timestamp <= endTime; timestamp = timestamp + intervalDuration) {
+        buckets.push(timestamp);
+      }
+
+      const resultBucketMap: Map<number, MetricTimeseriesInterval> = new Map(
+        results
+          .map(result => this.resultToTimeseriesInterval(result, spec))
+          .map(metric => [metric.timestamp.getTime(), metric])
+      );
+
+      const metrics = buckets.map(
+        timestamp =>
+          resultBucketMap.get(timestamp) ?? {
+            value: 0,
+            timestamp: new Date(timestamp)
+          }
+      );
+
+      return metrics;
+    }
+
+    return results.map(result => this.resultToTimeseriesInterval(result, spec));
+  }
+
   private resultToTimeseriesInterval(
     result: GraphQlExploreResult,
     spec: ExploreSpecification