Add Graceful Shutdown Support

packages/opentelemetry-core/src/platform/browser/ShutdownNotifier.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export function notifyOnGlobalShutdown(cb: () => void): () => void {
+  window.addEventListener('unload', cb, { once: true });
+  return function removeCallbackFromGlobalShutdown() {
+    window.removeEventListener('unload', cb, false);
+  };
+}
+
+export function _invokeGlobalShutdown() {
+  window.close();
+}

packages/opentelemetry-core/src/platform/browser/index.ts

@@ -21,3 +21,4 @@ export * from './RandomIdGenerator';
 export * from './performance';
 export * from './sdk-info';
 export * from './timer-util';
+export * from './ShutdownNotifier';

packages/opentelemetry-core/src/platform/node/ShutdownNotifier.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export function notifyOnGlobalShutdown(cb: () => void): () => void {
+  process.once('SIGTERM', cb);
+  return function removeCallbackFromGlobalShutdown() {
+    process.removeListener('SIGTERM', cb);
+  };
+}
+
+export function _invokeGlobalShutdown() {
+  process.kill(process.pid, 'SIGTERM');
+}

packages/opentelemetry-core/src/platform/node/index.ts

@@ -21,3 +21,4 @@ export * from './RandomIdGenerator';
 export * from './performance';
 export * from './sdk-info';
 export * from './timer-util';
+export * from './ShutdownNotifier';

packages/opentelemetry-exporter-prometheus/test/prometheus.test.ts

@@ -15,6 +15,10 @@
  */
 
 import { HrTime, ObserverResult } from '@opentelemetry/api';
+import {
+  notifyOnGlobalShutdown,
+  _invokeGlobalShutdown,
+} from '@opentelemetry/core';
 import {
   CounterMetric,
   SumAggregator,
@@ -27,6 +31,12 @@ import * as assert from 'assert';
 import * as http from 'http';
 import { PrometheusExporter } from '../src';
 
+function _cleanupGlobalShutdownListeners() {
+  if (typeof window === 'undefined') {
+    process.removeAllListeners('SIGTERM');
+  }
+}
+
 const mockedHrTime: HrTime = [1586347902211, 0];
 const mockedTimeMS = 1586347902211000;
 
@@ -185,15 +195,23 @@ describe('PrometheusExporter', () => {
 
   describe('export', () => {
     let exporter: PrometheusExporter;
+    let meterProvider: MeterProvider;
     let meter: Meter;
 
     beforeEach(done => {
       exporter = new PrometheusExporter();
-      meter = new MeterProvider().getMeter('test-prometheus');
+      meterProvider = new MeterProvider({
+        interval: Math.pow(2, 31) - 1,
+        gracefulShutdown: true,
+      });
+      meter = meterProvider.getMeter('test-prometheus', '1', {
+        exporter: exporter,
+      });
       exporter.startServer(done);
     });
 
     afterEach(done => {
+      _cleanupGlobalShutdownListeners();
       exporter.shutdown(done);
     });
 
@@ -320,6 +338,70 @@ describe('PrometheusExporter', () => {
       });
     });
 
+    it('should export multiple labels on graceful shutdown', done => {
+      const counter = meter.createCounter('counter', {
+        description: 'a test description',
+      }) as CounterMetric;
+
+      counter.bind({ counterKey1: 'labelValue1' }).add(10);
+      counter.bind({ counterKey1: 'labelValue2' }).add(20);
+      counter.bind({ counterKey1: 'labelValue3' }).add(30);
+
+      notifyOnGlobalShutdown(() => {
+        http
+          .get('http://localhost:9464/metrics', res => {
+            res.on('data', chunk => {
+              const body = chunk.toString();
+              const lines = body.split('\n');
+
+              assert.deepStrictEqual(lines, [
+                '# HELP counter a test description',
+                '# TYPE counter counter',
+                `counter{counterKey1="labelValue1"} 10 ${mockedTimeMS}`,
+                `counter{counterKey1="labelValue2"} 20 ${mockedTimeMS}`,
+                `counter{counterKey1="labelValue3"} 30 ${mockedTimeMS}`,
+                '',
+              ]);
+
+              done();
+            });
+          })
+          .on('error', errorHandler(done));
+      });
+      _invokeGlobalShutdown();
+    });
+
+    it('should export multiple labels on manual shutdown', done => {
+      const counter = meter.createCounter('counter', {
+        description: 'a test description',
+      }) as CounterMetric;
+
+      counter.bind({ counterKey1: 'labelValue1' }).add(10);
+      counter.bind({ counterKey1: 'labelValue2' }).add(20);
+      counter.bind({ counterKey1: 'labelValue3' }).add(30);
+      meterProvider.shutdown(() => {
+        http
+          .get('http://localhost:9464/metrics', res => {
+            res.on('data', chunk => {
+              const body = chunk.toString();
+              const lines = body.split('\n');
+
+              assert.deepStrictEqual(lines, [
+                '# HELP counter a test description',
+                '# TYPE counter counter',
+                `counter{counterKey1="labelValue1"} 10 ${mockedTimeMS}`,
+                `counter{counterKey1="labelValue2"} 20 ${mockedTimeMS}`,
+                `counter{counterKey1="labelValue3"} 30 ${mockedTimeMS}`,
+                '',
+              ]);
+
+              done();
+            });
+          })
+          .on('error', errorHandler(done));
+      });
+    });
+
     it('should export a comment if no metrics are registered', done => {
       exporter.export([], () => {
         http
@@ -426,6 +508,7 @@ describe('PrometheusExporter', () => {
 
     beforeEach(() => {
       meter = new MeterProvider().getMeter('test-prometheus');
+      process.removeAllListeners('SIGTERM');
       counter = meter.createCounter('counter') as CounterMetric;
       counter.bind({ key1: 'labelValue1' }).add(10);
     });

packages/opentelemetry-metrics/src/Meter.ts

@@ -40,6 +40,7 @@ export class Meter implements api.Meter {
   private readonly _batcher: Batcher;
   private readonly _resource: Resource;
   private readonly _instrumentationLibrary: InstrumentationLibrary;
+  private readonly _controller: PushController;
 
   /**
    * Constructs a new Meter instance.
@@ -55,7 +56,7 @@ export class Meter implements api.Meter {
     // start the push controller
     const exporter = config.exporter || new NoopExporter();
     const interval = config.interval;
-    new PushController(this, exporter, interval);
+    this._controller = new PushController(this, exporter, interval);
   }
 
   /**
@@ -309,6 +310,10 @@ export class Meter implements api.Meter {
     return this._batcher;
   }
 
+  async shutdown(): Promise<void> {
+    await this._controller.shutdown();
+  }
+
   /**
    * Registers metric to register.
    * @param name The name of the metric.

packages/opentelemetry-metrics/src/MeterProvider.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { ConsoleLogger } from '@opentelemetry/core';
+import { ConsoleLogger, notifyOnGlobalShutdown } from '@opentelemetry/core';
 import * as api from '@opentelemetry/api';
 import { Resource } from '@opentelemetry/resources';
 import { Meter } from '.';
@@ -36,6 +36,9 @@ export class MeterProvider implements api.MeterProvider {
       logger: this.logger,
       resource: this.resource,
     });
+    if (this._config.gracefulShutdown) {
+      notifyOnGlobalShutdown(this._shutdownAllMeters.bind(this));
+    }
   }
 
   /**
@@ -54,4 +57,19 @@ export class MeterProvider implements api.MeterProvider {
 
     return this._meters.get(key)!;
   }
+
+  shutdown(cb: () => void = () => {}): void {
+    this._shutdownAllMeters().then(() => {
+      setTimeout(cb, 0);
+    });
+  }
+
+  private async _shutdownAllMeters() {
+    if (this._config.exporter) {
+      this._config.exporter.shutdown();
+    }
+    await Promise.all(
+      Array.from(this._meters, ([_, meter]) => meter.shutdown())
+    );
+  }
 }

packages/opentelemetry-metrics/src/export/Controller.ts

@@ -38,12 +38,19 @@ export class PushController extends Controller {
     unrefTimer(this._timer);
   }
 
-  private _collect() {
-    this._meter.collect();
-    this._exporter.export(this._meter.getBatcher().checkPointSet(), result => {
-      if (result !== ExportResult.SUCCESS) {
-        // @todo: log error
+  async shutdown(): Promise<void> {
+    await this._collect();
+  }
+
+  private async _collect() {
+    await this._meter.collect();
+    await this._exporter.export(
+      this._meter.getBatcher().checkPointSet(),
+      result => {
+        if (result !== ExportResult.SUCCESS) {
+          // @todo: log error
+        }
       }
-    });
+    );
   }
 }

packages/opentelemetry-metrics/src/types.ts

@@ -39,11 +39,15 @@ export interface MeterConfig {
 
   /** Metric batcher. */
   batcher?: Batcher;
+
+  /** Bool for whether or not graceful shutdown is enabled */
+  gracefulShutdown?: boolean;
 }
 
 /** Default Meter configuration. */
 export const DEFAULT_CONFIG = {
   logLevel: getEnv().OTEL_LOG_LEVEL,
+  gracefulShutdown: true,
 };
 
 /** The default metric creation options value. */

packages/opentelemetry-metrics/test/MeterProvider.test.ts

@@ -15,10 +15,25 @@
  */
 
 import * as assert from 'assert';
+import * as sinon from 'sinon';
 import { MeterProvider, Meter, CounterMetric } from '../src';
-import { NoopLogger } from '@opentelemetry/core';
+import {
+  NoopLogger,
+  notifyOnGlobalShutdown,
+  _invokeGlobalShutdown,
+} from '@opentelemetry/core';
+
+function _cleanupGlobalShutdownListeners() {
+  if (typeof window === 'undefined') {
+    process.removeAllListeners('SIGTERM');
+  }
+}
 
 describe('MeterProvider', () => {
+  afterEach(() => {
+    _cleanupGlobalShutdownListeners();
+  });
+
   describe('constructor', () => {
     it('should construct an instance without any options', () => {
       const provider = new MeterProvider();
@@ -73,4 +88,45 @@ describe('MeterProvider', () => {
       assert.notEqual(meter3, meter4);
     });
   });
+
+  describe('shutdown()', () => {
+    it('should call shutdown when SIGTERM is received', () => {
+      const meterProvider = new MeterProvider({
+        interval: Math.pow(2, 31) - 1,
+        gracefulShutdown: true,
+      });
+      const sandbox = sinon.createSandbox();
+      const shutdownStub1 = sandbox.stub(
+        meterProvider.getMeter('meter1'),
+        'shutdown'
+      );
+      const shutdownStub2 = sandbox.stub(
+        meterProvider.getMeter('meter2'),
+        'shutdown'
+      );
+      notifyOnGlobalShutdown(() => {
+        sinon.assert.calledOnce(shutdownStub1);
+        sinon.assert.calledOnce(shutdownStub2);
+        sandbox.restore();
+      });
+      _invokeGlobalShutdown();
+    });
+
+    it('should not trigger shutdown if graceful shutdown is turned off', () => {
+      const meterProvider = new MeterProvider({
+        interval: Math.pow(2, 31) - 1,
+        gracefulShutdown: false,
+      });
+      const sandbox = sinon.createSandbox();
+      const shutdownStub = sandbox.stub(
+        meterProvider.getMeter('meter1'),
+        'shutdown'
+      );
+      notifyOnGlobalShutdown(() => {
+        sinon.assert.notCalled(shutdownStub);
+        sandbox.restore();
+      });
+      _invokeGlobalShutdown();
+    });
+  });
 });