From 579807ccb0f496a008aae2867f1ff2db1b220880 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Fri, 23 Dec 2022 03:42:08 -0800 Subject: [PATCH 01/15] Initial commit --- examples/agent/index.js | 14 ++++++++-- src/index.ts | 1 + src/logger/MetricValues.ts | 5 +++- src/logger/MetricsContext.ts | 8 +++--- src/logger/MetricsLogger.ts | 6 +++-- src/logger/StorageResolution.ts | 19 +++++++++++++ src/logger/__tests__/MetricsContext.test.ts | 17 ++++++++++++ src/serializers/LogSerializer.ts | 21 +++++++++++---- src/utils/Validator.ts | 30 ++++++++++++++++++++- 9 files changed, 107 insertions(+), 14 deletions(-) create mode 100644 src/logger/StorageResolution.ts diff --git a/examples/agent/index.js b/examples/agent/index.js index 24ae933..5f91149 100644 --- a/examples/agent/index.js +++ b/examples/agent/index.js @@ -1,9 +1,19 @@ -const { metricScope, Unit } = require('aws-embedded-metrics'); +const { metricScope, Unit, StorageResolution} = require('aws-embedded-metrics'); const doWork = metricScope(metrics => async event => { metrics.putDimensions({ Operation: 'Agent' }); - metrics.putMetric('ExampleMetric', 100, Unit.Milliseconds); + metrics.putMetric('ExampleMetricHigh', 100, Unit.Milliseconds); + metrics.putMetric('ExampleMetricHigh', 101, Unit.Milliseconds,StorageResolution.High); + metrics.setProperty('RequestId', '422b1569-16f6-4a03-b8f0-fe3fd9b100f8'); +}); + +const dodummyWork = metricScope(metrics => async event => { + metrics.putDimensions({ Operation: 'Agent' }); + metrics.putMetric('ExampleMetricHigh', 102, Unit.Milliseconds, StorageResolution.High); + metrics.putMetric('ExampleMetricHigh', 103, Unit.Milliseconds, StorageResolution.High); metrics.setProperty('RequestId', '422b1569-16f6-4a03-b8f0-fe3fd9b100f8'); }); doWork(); +// dodummyWork(); +// doWork(); diff --git a/src/index.ts b/src/index.ts index 883c260..397f204 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,6 +19,7 @@ export { AgentSink } from './sinks/AgentSink'; export { metricScope } from './logger/MetricScope'; export { createMetricsLogger } from './logger/MetricsLoggerFactory'; export { Unit } from './logger/Unit'; +export { StorageResolution } from './logger/StorageResolution'; import Configuration from './config/Configuration'; export { Configuration }; diff --git a/src/logger/MetricValues.ts b/src/logger/MetricValues.ts index 45ed89c..ef4bcfc 100644 --- a/src/logger/MetricValues.ts +++ b/src/logger/MetricValues.ts @@ -14,14 +14,17 @@ */ import { Unit } from '..'; +import { StorageResolution } from './StorageResolution'; export class MetricValues { public values: number[]; public unit: string; + public storageResolution: StorageResolution; - constructor(value: number, unit?: Unit | string) { + constructor(value: number, unit?: Unit | string, storageResolution?: StorageResolution) { this.values = [value]; this.unit = unit || 'None'; + this.storageResolution = storageResolution || StorageResolution.Standard; } /** diff --git a/src/logger/MetricsContext.ts b/src/logger/MetricsContext.ts index b8c9ecf..f2a29df 100644 --- a/src/logger/MetricsContext.ts +++ b/src/logger/MetricsContext.ts @@ -17,6 +17,7 @@ import Configuration from '../config/Configuration'; import { LOG } from '../utils/Logger'; import { validateNamespace, validateTimestamp, validateDimensionSet, validateMetric } from '../utils/Validator'; import { MetricValues } from './MetricValues'; +import { StorageResolution } from './StorageResolution'; import { Unit } from './Unit'; interface IProperties { @@ -41,6 +42,7 @@ export class MetricsContext { private defaultDimensions: Record; private shouldUseDefaultDimensions = true; private timestamp: Date | number | undefined; + private metricNameAndResolutionMap: Map = new Map(); /** * Constructor used to create child instances. @@ -180,14 +182,14 @@ export class MetricsContext { }); } - public putMetric(key: string, value: number, unit?: Unit | string): void { - validateMetric(key, value, unit); + public putMetric(key: string, value: number, unit?: Unit | string, storageResolution?:StorageResolution): void { + validateMetric(key, value, unit, storageResolution, this.metricNameAndResolutionMap); const currentMetric = this.metrics.get(key); if (currentMetric) { currentMetric.addValue(value); } else { - this.metrics.set(key, new MetricValues(value, unit)); + this.metrics.set(key, new MetricValues(value, unit, storageResolution)); } } diff --git a/src/logger/MetricsLogger.ts b/src/logger/MetricsLogger.ts index 64f20f6..48b7447 100644 --- a/src/logger/MetricsLogger.ts +++ b/src/logger/MetricsLogger.ts @@ -18,6 +18,7 @@ import { EnvironmentProvider } from '../environment/EnvironmentDetector'; import { IEnvironment } from '../environment/IEnvironment'; import { MetricsContext } from './MetricsContext'; import { Unit } from './Unit'; +import { StorageResolution } from './StorageResolution'; /** * An async metrics logger. @@ -124,9 +125,10 @@ export class MetricsLogger { * @param key * @param value * @param unit + * @param storageResolution */ - public putMetric(key: string, value: number, unit?: Unit | string): MetricsLogger { - this.context.putMetric(key, value, unit); + public putMetric(key: string, value: number, unit?: Unit | string, storageResolution?: StorageResolution): MetricsLogger { + this.context.putMetric(key, value, unit, storageResolution); return this; } diff --git a/src/logger/StorageResolution.ts b/src/logger/StorageResolution.ts new file mode 100644 index 0000000..b15bcf6 --- /dev/null +++ b/src/logger/StorageResolution.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export enum StorageResolution { + High = 1, + Standard = 60, +} diff --git a/src/logger/__tests__/MetricsContext.test.ts b/src/logger/__tests__/MetricsContext.test.ts index a3dec21..1b5c40f 100644 --- a/src/logger/__tests__/MetricsContext.test.ts +++ b/src/logger/__tests__/MetricsContext.test.ts @@ -203,6 +203,23 @@ test('putMetric uses None unit if not provided', () => { expect(metricDatum.unit).toBe(expectedUnit); }); +test('putMetric uses Standard storageResolution if not provided', () => { + // arrange + const context = MetricsContext.empty(); + const expectedKey = faker.random.word(); + const expectedValue = faker.datatype.number(); + const expectedStorageResolution = 60; //TODO_M remove this after metric validation + + // act + context.putMetric(expectedKey, expectedValue); + + // assert + const metricDatum: any = context.metrics.get(expectedKey); + expect(metricDatum).toBeTruthy(); + expect(metricDatum.values).toStrictEqual([expectedValue]); + expect(metricDatum.storageResolution).toBe(expectedStorageResolution); +}); + test('createCopyWithContext creates new instance', () => { // arrange const context = MetricsContext.empty(); diff --git a/src/serializers/LogSerializer.ts b/src/serializers/LogSerializer.ts index 2bae253..6561ee6 100644 --- a/src/serializers/LogSerializer.ts +++ b/src/serializers/LogSerializer.ts @@ -17,6 +17,7 @@ import { MaxHeap } from '@datastructures-js/heap'; import { Constants } from '../Constants'; import { DimensionSetExceededError } from '../exceptions/DimensionSetExceededError'; import { MetricsContext } from '../logger/MetricsContext'; +import { StorageResolution } from '../logger/StorageResolution'; import { ISerializer } from './Serializer'; interface MetricProgress { @@ -38,7 +39,7 @@ export class LogSerializer implements ISerializer { const dimensionKeys: string[][] = []; let dimensionProperties = {}; - context.getDimensions().forEach(dimensionSet => { + context.getDimensions().forEach((dimensionSet) => { const keys = Object.keys(dimensionSet); if (keys.length > Constants.MAX_DIMENSION_SET_SIZE) { @@ -89,7 +90,7 @@ export class LogSerializer implements ISerializer { Array.from(context.metrics, ([key, value]) => { return { name: key, numLeft: value.values.length }; }), - metric => metric.numLeft, + (metric) => metric.numLeft, ); let processedMetrics: MetricProgress[] = []; @@ -109,8 +110,18 @@ export class LogSerializer implements ISerializer { // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access currentBody[metricProgress.name] = metricValue; // eslint-disable-next-line - currentBody._aws.CloudWatchMetrics[0].Metrics.push({ Name: metricProgress.name, Unit: metric.unit }); - + if (metric.storageResolution === StorageResolution.Standard) { + currentBody._aws.CloudWatchMetrics[0].Metrics.push({ + Name: metricProgress.name, + Unit: metric.unit, + }); + } else { + currentBody._aws.CloudWatchMetrics[0].Metrics.push({ + Name: metricProgress.name, + Unit: metric.unit, + StorageResolution: metric.storageResolution, + }); + } metricProgress.numLeft -= Constants.MAX_VALUES_PER_METRIC; if (metricProgress.numLeft > 0) { processedMetrics.push(metricProgress); @@ -119,7 +130,7 @@ export class LogSerializer implements ISerializer { if (hasMaxMetrics() || remainingMetrics.isEmpty()) { serializeCurrentBody(); // inserts these metrics back in the heap to be processed in the next iteration. - processedMetrics.forEach(processingMetric => remainingMetrics.insert(processingMetric)); + processedMetrics.forEach((processingMetric) => remainingMetrics.insert(processingMetric)); processedMetrics = []; } } diff --git a/src/utils/Validator.ts b/src/utils/Validator.ts index 7f3ad99..e54f0d4 100644 --- a/src/utils/Validator.ts +++ b/src/utils/Validator.ts @@ -16,6 +16,7 @@ import validator from 'validator'; import { Constants } from '../Constants'; import { Unit } from '../logger/Unit'; +import { StorageResolution } from '../logger/StorageResolution'; import { DimensionSetExceededError } from '../exceptions/DimensionSetExceededError'; import { InvalidDimensionError } from '../exceptions/InvalidDimensionError'; import { InvalidMetricError } from '../exceptions/InvalidMetricError'; @@ -80,10 +81,18 @@ const validateDimensionSet = (dimensionSet: Record): void => { * * @param key * @param value + * @param unit + * @param storageResolution * * @throws {InvalidMetricError} Metric name must be valid. */ -const validateMetric = (key: string, value: number, unit?: Unit | string): void => { +const validateMetric = ( + key: string, + value: number, + unit?: Unit | string, + storageResolution?: StorageResolution, + metricNameAndResolutionMap?: Map, +): void => { if (key.trim().length == 0) { throw new InvalidMetricError(`Metric key ${key} must include at least one non-whitespace character`); } @@ -116,6 +125,25 @@ const validateMetric = (key: string, value: number, unit?: Unit | string): void ) { throw new InvalidMetricError(`Metric unit ${unit} is not valid`); } + + if ( + storageResolution !== undefined && + !Object.values(StorageResolution) + .map((s) => s) + .includes(storageResolution) + ) { + throw new InvalidMetricError(`Metric resolution ${storageResolution} is not valid`); + } + + if (metricNameAndResolutionMap && metricNameAndResolutionMap.has(key)) { + if (metricNameAndResolutionMap.get(key) !== (storageResolution?storageResolution:StorageResolution.Standard)) { + throw new InvalidMetricError( + `Resolution for metrics ${key} is already set. A single log event cannot have a metric with two different resolutions.`, + ); + } + } else { + metricNameAndResolutionMap?.set(key, storageResolution||StorageResolution.Standard); + } }; /** From eb3060e0ff11aaa0404b927e5ac5fb6dcfd077b5 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Mon, 2 Jan 2023 19:52:16 -0800 Subject: [PATCH 02/15] Added unit tests for high resolution metrics --- examples/agent/index.js | 14 ++--- src/logger/__tests__/MetricsContext.test.ts | 36 ++++++++++++- src/logger/__tests__/MetricsLogger.test.ts | 53 ++++++++++++++++++- src/serializers/LogSerializer.ts | 20 +++---- .../__tests__/LogSerializer.test.ts | 50 +++++++++++++++++ 5 files changed, 148 insertions(+), 25 deletions(-) diff --git a/examples/agent/index.js b/examples/agent/index.js index 5f91149..3057e4c 100644 --- a/examples/agent/index.js +++ b/examples/agent/index.js @@ -2,18 +2,10 @@ const { metricScope, Unit, StorageResolution} = require('aws-embedded-metrics'); const doWork = metricScope(metrics => async event => { metrics.putDimensions({ Operation: 'Agent' }); - metrics.putMetric('ExampleMetricHigh', 100, Unit.Milliseconds); - metrics.putMetric('ExampleMetricHigh', 101, Unit.Milliseconds,StorageResolution.High); - metrics.setProperty('RequestId', '422b1569-16f6-4a03-b8f0-fe3fd9b100f8'); -}); - -const dodummyWork = metricScope(metrics => async event => { - metrics.putDimensions({ Operation: 'Agent' }); - metrics.putMetric('ExampleMetricHigh', 102, Unit.Milliseconds, StorageResolution.High); - metrics.putMetric('ExampleMetricHigh', 103, Unit.Milliseconds, StorageResolution.High); + metrics.putMetric('ExampleMetric', 100, Unit.Milliseconds); + metrics.putMetric('ExampleMetric', 101, Unit.Milliseconds); + metrics.putMetric('ExampleHighResolutionMetric', 11, Unit.Milliseconds, StorageResolution.High); metrics.setProperty('RequestId', '422b1569-16f6-4a03-b8f0-fe3fd9b100f8'); }); doWork(); -// dodummyWork(); -// doWork(); diff --git a/src/logger/__tests__/MetricsContext.test.ts b/src/logger/__tests__/MetricsContext.test.ts index 1b5c40f..8bfeda1 100644 --- a/src/logger/__tests__/MetricsContext.test.ts +++ b/src/logger/__tests__/MetricsContext.test.ts @@ -7,6 +7,7 @@ import { Unit } from '../Unit'; import { Constants } from '../../Constants'; import { InvalidNamespaceError } from '../../exceptions/InvalidNamespaceError'; import { InvalidTimestampError } from '../../exceptions/InvalidTimestampError'; +import { StorageResolution } from '../StorageResolution'; test('can set property', () => { // arrange @@ -169,7 +170,7 @@ test('getDimensions returns only custom dimensions if no default dimensions', () expect(dimensions[0]).toStrictEqual(expectedDimensions); }); -test('putMetric adds metric to metrics key', () => { +test('putMetric adds standard resolution metric to metrics key', () => { // arrange const context = MetricsContext.empty(); const expectedKey = faker.random.word(); @@ -186,6 +187,25 @@ test('putMetric adds metric to metrics key', () => { expect(metricDatum.unit).toBe(expectedUnit); }); +test('putMetric adds high resolution metric to metrics key', () => { + // arrange + const context = MetricsContext.empty(); + const expectedKey = faker.random.word(); + const expectedValue = faker.datatype.number(); + const expectedUnit = faker.helpers.arrayElement(Object.values(Unit)); + const expectedStorageResolution = StorageResolution.High; + + // act + context.putMetric(expectedKey, expectedValue, expectedUnit, expectedStorageResolution); + + // assert + const metricDatum: any = context.metrics.get(expectedKey); + expect(metricDatum).toBeTruthy(); + expect(metricDatum.values).toStrictEqual([expectedValue]); + expect(metricDatum.unit).toBe(expectedUnit); + expect(metricDatum.storageResolution).toBe(expectedStorageResolution); +}); + test('putMetric uses None unit if not provided', () => { // arrange const context = MetricsContext.empty(); @@ -370,6 +390,20 @@ test.each([ }).not.toThrow(InvalidMetricError); }); +test('put metric with same key and different resolution in single flush throws error' ,() => { + //arrange + const context = MetricsContext.empty(); + const expectedKey = 'MetricName'; + const expectedValue = faker.datatype.number(); + const expectedUnit = 'None'; + + // act + expect(() => { + context.putMetric(expectedKey, expectedValue,expectedUnit,StorageResolution.High); + context.putMetric(expectedKey, expectedValue,expectedUnit,StorageResolution.Standard); + }).toThrow(InvalidMetricError); +}); + test.each([[''], [' '], ['a'.repeat(Constants.MAX_NAMESPACE_LENGTH + 1)], ['àẁş/ćļốṹḓⱳầƭḉⱨ'], ['namespace ']])( 'setNamespace with invalid namespace: %s throws error', (namespace) => { diff --git a/src/logger/__tests__/MetricsLogger.test.ts b/src/logger/__tests__/MetricsLogger.test.ts index ffc3229..4417153 100644 --- a/src/logger/__tests__/MetricsLogger.test.ts +++ b/src/logger/__tests__/MetricsLogger.test.ts @@ -1,5 +1,5 @@ import { faker } from '@faker-js/faker'; -import { Unit } from '../..'; +import { Unit, StorageResolution } from '../..'; import { TestSink } from '../../../test/utils/TestSink'; import Configuration from '../../config/Configuration'; import { EnvironmentProvider } from '../../environment/EnvironmentDetector'; @@ -7,6 +7,7 @@ import { IEnvironment } from '../../environment/IEnvironment'; import { ISink } from '../../sinks/Sink'; import { MetricsContext } from '../MetricsContext'; import { MetricsLogger } from '../MetricsLogger'; +import { InvalidMetricError } from '../../exceptions/InvalidMetricError'; const createSink = (forceAcceptRejects = false) => new TestSink(forceAcceptRejects); const createEnvironment = (sink: ISink) => { @@ -117,6 +118,56 @@ describe('successful', () => { expect(actualMetric!.unit).toBe(expectedUnit); }); + test('can put high resolution metric with unit', async () => { + // arrange + const expectedKey = faker.random.word(); + const expectedValue = faker.datatype.number(); + const expectedUnit = Unit.Bits; + const expectedStorageResolution = StorageResolution.High; + + // act + logger.putMetric(expectedKey, expectedValue, expectedUnit, expectedStorageResolution); + await logger.flush(); + + // assert + expect(sink.events).toHaveLength(1); + const actualMetric = sink.events[0].metrics.get(expectedKey); + expect(actualMetric).toBeTruthy(); + expect(actualMetric!.unit).toBe(expectedUnit); + expect(actualMetric!.storageResolution).toBe(expectedStorageResolution); + }); + + test('can put high resolution metric without unit', async () => { + // arrange + const expectedKey = faker.random.word(); + const expectedValue = faker.datatype.number(); + const expectedStorageResolution = StorageResolution.High; + + // act + logger.putMetric(expectedKey, expectedValue, undefined, expectedStorageResolution); + await logger.flush(); + + // assert + expect(sink.events).toHaveLength(1); + const actualMetric = sink.events[0].metrics.get(expectedKey); + expect(actualMetric).toBeTruthy(); + expect(actualMetric!.storageResolution).toBe(expectedStorageResolution); + }); + + test('can put metric with same key and different resolution in separate flushes', async () => { + const expectedKey = 'MetricName'; + const expectedValue = faker.datatype.number(); + const expectedUnit = 'None'; + + // assert + expect(async () => { + logger.putMetric(expectedKey, expectedValue,expectedUnit,StorageResolution.Standard); + await logger.flush(); + logger.putMetric(expectedKey, expectedValue,expectedUnit,StorageResolution.High); + await logger.flush(); + }).not.toThrow(InvalidMetricError); + }); + test('can put dimension', async () => { // arrange const expectedKey = faker.random.word(); diff --git a/src/serializers/LogSerializer.ts b/src/serializers/LogSerializer.ts index 6561ee6..803672e 100644 --- a/src/serializers/LogSerializer.ts +++ b/src/serializers/LogSerializer.ts @@ -109,19 +109,15 @@ export class LogSerializer implements ISerializer { metric.values.slice(startIndex, startIndex + Constants.MAX_VALUES_PER_METRIC); // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access currentBody[metricProgress.name] = metricValue; - // eslint-disable-next-line - if (metric.storageResolution === StorageResolution.Standard) { - currentBody._aws.CloudWatchMetrics[0].Metrics.push({ - Name: metricProgress.name, - Unit: metric.unit, - }); - } else { - currentBody._aws.CloudWatchMetrics[0].Metrics.push({ - Name: metricProgress.name, - Unit: metric.unit, - StorageResolution: metric.storageResolution, - }); + let metricBody: {[key: string]: any} = { + Name: metricProgress.name, + Unit: metric.unit, } + if (metric.storageResolution === StorageResolution.High) + { + metricBody.StorageResolution = StorageResolution.High; + } + currentBody._aws.CloudWatchMetrics[0].Metrics.push(metricBody); metricProgress.numLeft -= Constants.MAX_VALUES_PER_METRIC; if (metricProgress.numLeft > 0) { processedMetrics.push(metricProgress); diff --git a/src/serializers/__tests__/LogSerializer.test.ts b/src/serializers/__tests__/LogSerializer.test.ts index f0ba6fa..f16b8e9 100644 --- a/src/serializers/__tests__/LogSerializer.test.ts +++ b/src/serializers/__tests__/LogSerializer.test.ts @@ -3,6 +3,7 @@ import { Constants } from '../../Constants'; import { MetricsContext } from '../../logger/MetricsContext'; import { LogSerializer } from '../LogSerializer'; import { DimensionSetExceededError } from '../../exceptions/DimensionSetExceededError'; +import { Unit, StorageResolution } from '../..'; test('serializes dimensions', () => { // arrange @@ -86,6 +87,55 @@ test('serializes metrics with multiple datapoints', () => { assertJsonEquality(resultJson, expected); }); +test('serialize high resolution metrics', () => { + // arrange + const expectedKey = faker.random.word(); + const expectedValue = faker.datatype.number(); + const expectedUnit = Unit.Bits; + const expectedStorageResolution = StorageResolution.High; + const expectedMetricDefinition = { + Name: expectedKey, + Unit: 'Bits', + StorageResolution:1 + }; + const expected: any = { ...getEmptyPayload() }; + expected[expectedKey] = expectedValue; + expected._aws.CloudWatchMetrics[0].Metrics.push(expectedMetricDefinition); + + const context = getContext(); + context.putMetric(expectedKey, expectedValue, expectedUnit,expectedStorageResolution); + + // act + const resultJson = serializer.serialize(context)[0]; + + // assert + assertJsonEquality(resultJson, expected); +}); + +test('serialize standard resolution metrics', () => { + // arrange + const expectedKey = faker.random.word(); + const expectedValue = faker.datatype.number(); + const expectedUnit = Unit.Bits; + const expectedStorageResolution = StorageResolution.Standard; + const expectedMetricDefinition = { + Name: expectedKey, + Unit: 'Bits' + }; + const expected: any = { ...getEmptyPayload() }; + expected[expectedKey] = expectedValue; + expected._aws.CloudWatchMetrics[0].Metrics.push(expectedMetricDefinition); + + const context = getContext(); + context.putMetric(expectedKey, expectedValue, expectedUnit,expectedStorageResolution); + + // act + const resultJson = serializer.serialize(context)[0]; + + // assert + assertJsonEquality(resultJson, expected); +}); + test('serializes more than 100 metrics into multiple events', () => { // arrange const expectedValue = 1; From 66b2dfb488b3591f424c07da95b446c471a9eec5 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Mon, 2 Jan 2023 20:29:02 -0800 Subject: [PATCH 03/15] Linting fix --- examples/agent/index.js | 3 +-- src/logger/__tests__/MetricsLogger.test.ts | 16 ++++++++-------- src/serializers/LogSerializer.ts | 3 ++- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/agent/index.js b/examples/agent/index.js index 3057e4c..7b40ae4 100644 --- a/examples/agent/index.js +++ b/examples/agent/index.js @@ -3,8 +3,7 @@ const { metricScope, Unit, StorageResolution} = require('aws-embedded-metrics'); const doWork = metricScope(metrics => async event => { metrics.putDimensions({ Operation: 'Agent' }); metrics.putMetric('ExampleMetric', 100, Unit.Milliseconds); - metrics.putMetric('ExampleMetric', 101, Unit.Milliseconds); - metrics.putMetric('ExampleHighResolutionMetric', 11, Unit.Milliseconds, StorageResolution.High); + metrics.putMetric('ExampleHighResolutionMetric', 101, Unit.Milliseconds, StorageResolution.High); metrics.setProperty('RequestId', '422b1569-16f6-4a03-b8f0-fe3fd9b100f8'); }); diff --git a/src/logger/__tests__/MetricsLogger.test.ts b/src/logger/__tests__/MetricsLogger.test.ts index 4417153..77d433d 100644 --- a/src/logger/__tests__/MetricsLogger.test.ts +++ b/src/logger/__tests__/MetricsLogger.test.ts @@ -154,18 +154,18 @@ describe('successful', () => { expect(actualMetric!.storageResolution).toBe(expectedStorageResolution); }); - test('can put metric with same key and different resolution in separate flushes', async () => { + test('can put metric with same key and different resolution in separate flushes', () => { const expectedKey = 'MetricName'; const expectedValue = faker.datatype.number(); const expectedUnit = 'None'; - + // assert - expect(async () => { - logger.putMetric(expectedKey, expectedValue,expectedUnit,StorageResolution.Standard); - await logger.flush(); - logger.putMetric(expectedKey, expectedValue,expectedUnit,StorageResolution.High); - await logger.flush(); - }).not.toThrow(InvalidMetricError); + expect(async () => { + logger.putMetric(expectedKey, expectedValue, expectedUnit, StorageResolution.Standard); + await logger.flush(); + logger.putMetric(expectedKey, expectedValue, expectedUnit, StorageResolution.High); + await logger.flush(); + }).not.toThrow(InvalidMetricError); }); test('can put dimension', async () => { diff --git a/src/serializers/LogSerializer.ts b/src/serializers/LogSerializer.ts index 803672e..f0161eb 100644 --- a/src/serializers/LogSerializer.ts +++ b/src/serializers/LogSerializer.ts @@ -109,7 +109,7 @@ export class LogSerializer implements ISerializer { metric.values.slice(startIndex, startIndex + Constants.MAX_VALUES_PER_METRIC); // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access currentBody[metricProgress.name] = metricValue; - let metricBody: {[key: string]: any} = { + const metricBody: {[key: string]: any} = { Name: metricProgress.name, Unit: metric.unit, } @@ -117,6 +117,7 @@ export class LogSerializer implements ISerializer { { metricBody.StorageResolution = StorageResolution.High; } + // eslint-disable-next-line currentBody._aws.CloudWatchMetrics[0].Metrics.push(metricBody); metricProgress.numLeft -= Constants.MAX_VALUES_PER_METRIC; if (metricProgress.numLeft > 0) { From 837243f9beb409a3d47b968e45ae5714ab211981 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Tue, 3 Jan 2023 01:00:32 -0800 Subject: [PATCH 04/15] Updated README.md and examples --- README.md | 18 +++++++++--------- examples/agent/index.js | 2 +- examples/ecs-firelens/app.js | 3 ++- examples/eks/app.js | 3 ++- examples/lambda/src/index.js | 3 ++- test/canary/agent/index.js | 4 ++-- test/integ/agent/end-to-end.integ.ts | 5 +++-- 7 files changed, 21 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 7463435..5356129 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,12 @@ To get a metric logger, you can either decorate your function with a metricScope Using the metricScope decorator without function parameters: ```js -const { metricScope, Unit } = require("aws-embedded-metrics"); +const { metricScope, Unit, StorageResolution } = require("aws-embedded-metrics"); const myFunc = metricScope(metrics => async () => { metrics.putDimensions({ Service: "Aggregator" }); - metrics.putMetric("ProcessingLatency", 100, Unit.Milliseconds); + metrics.putMetric("ProcessingLatency", 100, Unit.Milliseconds, StorageResolution.Standard); metrics.setProperty("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8"); // ... }); @@ -54,12 +54,12 @@ await myFunc(); Using the metricScope decorator with function parameters: ```js -const { metricScope, Unit } = require("aws-embedded-metrics"); +const { metricScope, Unit, StorageResolution } = require("aws-embedded-metrics"); const myFunc = metricScope(metrics => async (param1: string, param2: number) => { metrics.putDimensions({ Service: "Aggregator" }); - metrics.putMetric("ProcessingLatency", 100, Unit.Milliseconds); + metrics.putMetric("ProcessingLatency", 100, Unit.Milliseconds, StorageResolution.Standard); metrics.setProperty("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8"); // ... }); @@ -70,12 +70,12 @@ await myFunc('myParam', 0); Manually constructing and flushing the logger: ```js -const { createMetricsLogger, Unit } = require("aws-embedded-metrics"); +const { createMetricsLogger, Unit, StorageResolution } = require("aws-embedded-metrics"); const myFunc = async () => { const metrics = createMetricsLogger(); metrics.putDimensions({ Service: "Aggregator" }); - metrics.putMetric("ProcessingLatency", 100, Unit.Milliseconds); + metrics.putMetric("ProcessingLatency", 100, Unit.Milliseconds), StorageResolution.Standard; metrics.setProperty("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8"); // ... await metrics.flush(); @@ -105,9 +105,9 @@ exports.handler = myFunc; The `MetricLogger` is the interface you will use to publish embedded metrics. -- **putMetric**(String name, Double value, Unit? unit) +- **putMetric**(String name, Double value, Unit? unit, StorageResolution? storageResolution) -Adds a new metric to the current logger context. Multiple metrics using the same key will be appended to an array of values. The Embedded Metric Format supports a maximum of 100 values per key. If more metric values are added than are supported by the format, the logger will be flushed to allow for new metric values to be captured. +Adds a new metric to the current logger context. Multiple metrics using the same key will be appended to an array of values. Multiple metrics cannot have same key and different storage resolution. The Embedded Metric Format supports a maximum of 100 values per key. If more metric values are added than are supported by the format, the logger will be flushed to allow for new metric values to be captured. Requirements: - Name Length 1-255 characters @@ -118,7 +118,7 @@ Requirements: Examples: ```js -putMetric("Latency", 200, Unit.Milliseconds) +putMetric("Latency", 200, Unit.Milliseconds, StorageResolution.Standard) ``` - **setProperty**(String key, Object value) diff --git a/examples/agent/index.js b/examples/agent/index.js index 7b40ae4..b7314da 100644 --- a/examples/agent/index.js +++ b/examples/agent/index.js @@ -3,7 +3,7 @@ const { metricScope, Unit, StorageResolution} = require('aws-embedded-metrics'); const doWork = metricScope(metrics => async event => { metrics.putDimensions({ Operation: 'Agent' }); metrics.putMetric('ExampleMetric', 100, Unit.Milliseconds); - metrics.putMetric('ExampleHighResolutionMetric', 101, Unit.Milliseconds, StorageResolution.High); + metrics.putMetric('ExampleHighResolutionMetric', 10, Unit.Milliseconds, StorageResolution.High); metrics.setProperty('RequestId', '422b1569-16f6-4a03-b8f0-fe3fd9b100f8'); }); diff --git a/examples/ecs-firelens/app.js b/examples/ecs-firelens/app.js index 69c942b..1002ec5 100644 --- a/examples/ecs-firelens/app.js +++ b/examples/ecs-firelens/app.js @@ -1,7 +1,7 @@ const Koa = require('koa'); const app = new Koa(); -const { metricScope, Unit } = require('aws-embedded-metrics'); +const { metricScope, Unit, StorageResolution } = require('aws-embedded-metrics'); app.use( metricScope(metrics => async (ctx, next) => { @@ -14,6 +14,7 @@ app.use( metrics.setProperty('Method', ctx.method); metrics.setProperty('Url', ctx.url); metrics.putMetric('ProcessingTime', Date.now() - start, Unit.Milliseconds); + metrics.putMetric('ProcessingLatency', 100, Unit.Milliseconds, StorageResolution.High); // send application logs to stdout, FireLens will send this to a different LogGroup console.log('Completed Request'); diff --git a/examples/eks/app.js b/examples/eks/app.js index 3f354bb..9c6a5aa 100644 --- a/examples/eks/app.js +++ b/examples/eks/app.js @@ -1,7 +1,7 @@ const Koa = require('koa'); const app = new Koa(); -const { metricScope, Configuration, Unit } = require('aws-embedded-metrics'); +const { metricScope, Configuration, Unit, StorageResolution } = require('aws-embedded-metrics'); Configuration.serviceName = 'EKS-Demo'; Configuration.serviceType = 'AWS::EKS::Cluster'; @@ -18,6 +18,7 @@ app.use( metrics.setProperty('Method', ctx.method); metrics.setProperty('Url', ctx.url); metrics.putMetric('ProcessingTime', Date.now() - start, Unit.Milliseconds); + metrics.putMetric('ProcessingLatency', 100, Unit.Milliseconds, StorageResolution.High); }), ); diff --git a/examples/lambda/src/index.js b/examples/lambda/src/index.js index 66331c7..2cd3bae 100644 --- a/examples/lambda/src/index.js +++ b/examples/lambda/src/index.js @@ -1,9 +1,10 @@ -const { metricScope } = require('aws-embedded-metrics'); +const { metricScope , StorageResolution} = require('aws-embedded-metrics'); const aggregator = metricScope(metrics => async event => { console.log('received message'); metrics.putDimensions({ Service: 'Aggregator' }); metrics.putMetric('ProcessingLatency', 100, 'Milliseconds'); + metrics.putMetric('CPU Utilization', 87, 'Percent', StorageResolution.High); metrics.setProperty('AccountId', '123456789012'); metrics.setProperty('RequestId', '422b1569-16f6-4a03-b8f0-fe3fd9b100f8'); metrics.setProperty('DeviceId', '61270781-c6ac-46f1-baf7-22c808af8162'); diff --git a/test/canary/agent/index.js b/test/canary/agent/index.js index 76df99d..5fdbeb9 100644 --- a/test/canary/agent/index.js +++ b/test/canary/agent/index.js @@ -1,4 +1,4 @@ -const { metricScope, Unit, Configuration } = require('aws-embedded-metrics'); +const { metricScope, Unit, Configuration, StorageResolution} = require('aws-embedded-metrics'); let version = ''; try { @@ -23,7 +23,7 @@ const recordMetric = metricScope(metrics => () => { metrics.putMetric('Invoke', 1, Unit.Count); metrics.putMetric('Memory.HeapTotal', memoryUsage.heapTotal, Unit.Bytes); - metrics.putMetric('Memory.HeapUsed', memoryUsage.heapUsed, Unit.Bytes); + metrics.putMetric('Memory.HeapUsed', memoryUsage.heapUsed, Unit.Bytes, StorageResolution.High); metrics.putMetric('Memory.RSS', memoryUsage.rss, Unit.Bytes); }); diff --git a/test/integ/agent/end-to-end.integ.ts b/test/integ/agent/end-to-end.integ.ts index 29f8857..feab454 100644 --- a/test/integ/agent/end-to-end.integ.ts +++ b/test/integ/agent/end-to-end.integ.ts @@ -1,6 +1,7 @@ import { metricScope } from '../../../src/logger/MetricScope'; import Sleep from '../../utils/Sleep'; import Configuration from '../../../src/config/Configuration'; +import { StorageResolution } from '../../../src'; import os = require('os'); import CloudWatch = require('aws-sdk/clients/cloudwatch'); const cwmClient = new CloudWatch(); @@ -33,7 +34,7 @@ test( const doWork = metricScope(metrics => () => { metrics.putDimensions(dimensions); - metrics.putMetric(metricName, 100, 'Milliseconds'); + metrics.putMetric(metricName, 100, 'Milliseconds', StorageResolution.High); }); // act @@ -57,7 +58,7 @@ test( const doWork = metricScope(metrics => () => { metrics.putDimensions(dimensions); - metrics.putMetric(metricName, 100, 'Milliseconds'); + metrics.putMetric(metricName, 100, 'Milliseconds', StorageResolution.High); metrics.setProperty('RequestId', '422b1569-16f6-4a03-b8f0-fe3fd9b100f8'); }); From 06644468b75169152767078d8d899d71f2fbef5e Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Tue, 3 Jan 2023 02:27:04 -0800 Subject: [PATCH 05/15] Minor fix --- examples/lambda/src/index.js | 2 +- src/logger/MetricValues.ts | 4 ++-- src/logger/MetricsContext.ts | 7 ++++++- src/logger/MetricsLogger.ts | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/lambda/src/index.js b/examples/lambda/src/index.js index 2cd3bae..a2303a3 100644 --- a/examples/lambda/src/index.js +++ b/examples/lambda/src/index.js @@ -4,7 +4,7 @@ const aggregator = metricScope(metrics => async event => { console.log('received message'); metrics.putDimensions({ Service: 'Aggregator' }); metrics.putMetric('ProcessingLatency', 100, 'Milliseconds'); - metrics.putMetric('CPU Utilization', 87, 'Percent', StorageResolution.High); + metrics.putMetric('CPU Utilization', 87, 'Percent', 1); metrics.setProperty('AccountId', '123456789012'); metrics.setProperty('RequestId', '422b1569-16f6-4a03-b8f0-fe3fd9b100f8'); metrics.setProperty('DeviceId', '61270781-c6ac-46f1-baf7-22c808af8162'); diff --git a/src/logger/MetricValues.ts b/src/logger/MetricValues.ts index ef4bcfc..bb0b39f 100644 --- a/src/logger/MetricValues.ts +++ b/src/logger/MetricValues.ts @@ -19,9 +19,9 @@ import { StorageResolution } from './StorageResolution'; export class MetricValues { public values: number[]; public unit: string; - public storageResolution: StorageResolution; + public storageResolution: number; - constructor(value: number, unit?: Unit | string, storageResolution?: StorageResolution) { + constructor(value: number, unit?: Unit | string, storageResolution?: StorageResolution | number) { this.values = [value]; this.unit = unit || 'None'; this.storageResolution = storageResolution || StorageResolution.Standard; diff --git a/src/logger/MetricsContext.ts b/src/logger/MetricsContext.ts index f2a29df..02c3299 100644 --- a/src/logger/MetricsContext.ts +++ b/src/logger/MetricsContext.ts @@ -182,7 +182,12 @@ export class MetricsContext { }); } - public putMetric(key: string, value: number, unit?: Unit | string, storageResolution?:StorageResolution): void { + public putMetric( + key: string, + value: number, + unit?: Unit | string, + storageResolution?: StorageResolution | number, + ): void { validateMetric(key, value, unit, storageResolution, this.metricNameAndResolutionMap); const currentMetric = this.metrics.get(key); diff --git a/src/logger/MetricsLogger.ts b/src/logger/MetricsLogger.ts index 48b7447..9eaa7be 100644 --- a/src/logger/MetricsLogger.ts +++ b/src/logger/MetricsLogger.ts @@ -127,7 +127,7 @@ export class MetricsLogger { * @param unit * @param storageResolution */ - public putMetric(key: string, value: number, unit?: Unit | string, storageResolution?: StorageResolution): MetricsLogger { + public putMetric(key: string, value: number, unit?: Unit | string, storageResolution?: StorageResolution | number): MetricsLogger { this.context.putMetric(key, value, unit, storageResolution); return this; } From 77c9b56cc0087761b42d797c64450a619107b73e Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Wed, 11 Jan 2023 08:44:07 -0800 Subject: [PATCH 06/15] minor fix --- src/logger/__tests__/MetricsContext.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/logger/__tests__/MetricsContext.test.ts b/src/logger/__tests__/MetricsContext.test.ts index 8bfeda1..ba893f3 100644 --- a/src/logger/__tests__/MetricsContext.test.ts +++ b/src/logger/__tests__/MetricsContext.test.ts @@ -228,7 +228,7 @@ test('putMetric uses Standard storageResolution if not provided', () => { const context = MetricsContext.empty(); const expectedKey = faker.random.word(); const expectedValue = faker.datatype.number(); - const expectedStorageResolution = 60; //TODO_M remove this after metric validation + const expectedStorageResolution = 60; // act context.putMetric(expectedKey, expectedValue); From df030d417f4549a412029ded453f2562b2fcb634 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Wed, 11 Jan 2023 09:02:36 -0800 Subject: [PATCH 07/15] Minor fixes --- examples/lambda/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/lambda/src/index.js b/examples/lambda/src/index.js index a2303a3..2cd3bae 100644 --- a/examples/lambda/src/index.js +++ b/examples/lambda/src/index.js @@ -4,7 +4,7 @@ const aggregator = metricScope(metrics => async event => { console.log('received message'); metrics.putDimensions({ Service: 'Aggregator' }); metrics.putMetric('ProcessingLatency', 100, 'Milliseconds'); - metrics.putMetric('CPU Utilization', 87, 'Percent', 1); + metrics.putMetric('CPU Utilization', 87, 'Percent', StorageResolution.High); metrics.setProperty('AccountId', '123456789012'); metrics.setProperty('RequestId', '422b1569-16f6-4a03-b8f0-fe3fd9b100f8'); metrics.setProperty('DeviceId', '61270781-c6ac-46f1-baf7-22c808af8162'); From 761cf7283c6def28b978fcbfedc73797a0f9d926 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Wed, 11 Jan 2023 14:49:50 -0800 Subject: [PATCH 08/15] Move Map updation from validator to metricsContext --- src/logger/MetricsContext.ts | 4 ++++ src/utils/Validator.ts | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/logger/MetricsContext.ts b/src/logger/MetricsContext.ts index 02c3299..042bc7f 100644 --- a/src/logger/MetricsContext.ts +++ b/src/logger/MetricsContext.ts @@ -196,6 +196,10 @@ export class MetricsContext { } else { this.metrics.set(key, new MetricValues(value, unit, storageResolution)); } + if (this.metricNameAndResolutionMap && !this.metricNameAndResolutionMap.has(key)) { + this.metricNameAndResolutionMap?.set(key, storageResolution||StorageResolution.Standard); + } + } /** diff --git a/src/utils/Validator.ts b/src/utils/Validator.ts index e54f0d4..5e21999 100644 --- a/src/utils/Validator.ts +++ b/src/utils/Validator.ts @@ -141,8 +141,6 @@ const validateMetric = ( `Resolution for metrics ${key} is already set. A single log event cannot have a metric with two different resolutions.`, ); } - } else { - metricNameAndResolutionMap?.set(key, storageResolution||StorageResolution.Standard); } }; From 9c8407630c397908f54dd462f41a4aeef27807ce Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Tue, 17 Jan 2023 05:52:47 -0800 Subject: [PATCH 09/15] prettier formatting --- examples/agent/index.js | 2 +- examples/lambda/src/index.js | 2 +- src/logger/MetricsContext.ts | 3 +-- src/logger/MetricsLogger.ts | 7 ++++++- src/logger/__tests__/MetricsContext.test.ts | 12 ++++++------ src/serializers/LogSerializer.ts | 7 +++---- src/serializers/__tests__/LogSerializer.test.ts | 8 ++++---- src/utils/Validator.ts | 2 +- test/canary/agent/index.js | 2 +- 9 files changed, 24 insertions(+), 21 deletions(-) diff --git a/examples/agent/index.js b/examples/agent/index.js index b7314da..b406baa 100644 --- a/examples/agent/index.js +++ b/examples/agent/index.js @@ -1,4 +1,4 @@ -const { metricScope, Unit, StorageResolution} = require('aws-embedded-metrics'); +const { metricScope, Unit, StorageResolution } = require('aws-embedded-metrics'); const doWork = metricScope(metrics => async event => { metrics.putDimensions({ Operation: 'Agent' }); diff --git a/examples/lambda/src/index.js b/examples/lambda/src/index.js index 2cd3bae..80cb6ff 100644 --- a/examples/lambda/src/index.js +++ b/examples/lambda/src/index.js @@ -1,4 +1,4 @@ -const { metricScope , StorageResolution} = require('aws-embedded-metrics'); +const { metricScope, StorageResolution } = require('aws-embedded-metrics'); const aggregator = metricScope(metrics => async event => { console.log('received message'); diff --git a/src/logger/MetricsContext.ts b/src/logger/MetricsContext.ts index 042bc7f..aa5183f 100644 --- a/src/logger/MetricsContext.ts +++ b/src/logger/MetricsContext.ts @@ -197,9 +197,8 @@ export class MetricsContext { this.metrics.set(key, new MetricValues(value, unit, storageResolution)); } if (this.metricNameAndResolutionMap && !this.metricNameAndResolutionMap.has(key)) { - this.metricNameAndResolutionMap?.set(key, storageResolution||StorageResolution.Standard); + this.metricNameAndResolutionMap?.set(key, storageResolution || StorageResolution.Standard); } - } /** diff --git a/src/logger/MetricsLogger.ts b/src/logger/MetricsLogger.ts index 9eaa7be..a362070 100644 --- a/src/logger/MetricsLogger.ts +++ b/src/logger/MetricsLogger.ts @@ -127,7 +127,12 @@ export class MetricsLogger { * @param unit * @param storageResolution */ - public putMetric(key: string, value: number, unit?: Unit | string, storageResolution?: StorageResolution | number): MetricsLogger { + public putMetric( + key: string, + value: number, + unit?: Unit | string, + storageResolution?: StorageResolution | number, + ): MetricsLogger { this.context.putMetric(key, value, unit, storageResolution); return this; } diff --git a/src/logger/__tests__/MetricsContext.test.ts b/src/logger/__tests__/MetricsContext.test.ts index ba893f3..cdec269 100644 --- a/src/logger/__tests__/MetricsContext.test.ts +++ b/src/logger/__tests__/MetricsContext.test.ts @@ -390,18 +390,18 @@ test.each([ }).not.toThrow(InvalidMetricError); }); -test('put metric with same key and different resolution in single flush throws error' ,() => { +test('put metric with same key and different resolution in single flush throws error', () => { //arrange const context = MetricsContext.empty(); const expectedKey = 'MetricName'; const expectedValue = faker.datatype.number(); const expectedUnit = 'None'; - // act - expect(() => { - context.putMetric(expectedKey, expectedValue,expectedUnit,StorageResolution.High); - context.putMetric(expectedKey, expectedValue,expectedUnit,StorageResolution.Standard); - }).toThrow(InvalidMetricError); + // act + expect(() => { + context.putMetric(expectedKey, expectedValue, expectedUnit, StorageResolution.High); + context.putMetric(expectedKey, expectedValue, expectedUnit, StorageResolution.Standard); + }).toThrow(InvalidMetricError); }); test.each([[''], [' '], ['a'.repeat(Constants.MAX_NAMESPACE_LENGTH + 1)], ['àẁş/ćļốṹḓⱳầƭḉⱨ'], ['namespace ']])( diff --git a/src/serializers/LogSerializer.ts b/src/serializers/LogSerializer.ts index f0161eb..1a5149c 100644 --- a/src/serializers/LogSerializer.ts +++ b/src/serializers/LogSerializer.ts @@ -109,12 +109,11 @@ export class LogSerializer implements ISerializer { metric.values.slice(startIndex, startIndex + Constants.MAX_VALUES_PER_METRIC); // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access currentBody[metricProgress.name] = metricValue; - const metricBody: {[key: string]: any} = { + const metricBody: { [key: string]: any } = { Name: metricProgress.name, Unit: metric.unit, - } - if (metric.storageResolution === StorageResolution.High) - { + }; + if (metric.storageResolution === StorageResolution.High) { metricBody.StorageResolution = StorageResolution.High; } // eslint-disable-next-line diff --git a/src/serializers/__tests__/LogSerializer.test.ts b/src/serializers/__tests__/LogSerializer.test.ts index f16b8e9..4d91203 100644 --- a/src/serializers/__tests__/LogSerializer.test.ts +++ b/src/serializers/__tests__/LogSerializer.test.ts @@ -96,14 +96,14 @@ test('serialize high resolution metrics', () => { const expectedMetricDefinition = { Name: expectedKey, Unit: 'Bits', - StorageResolution:1 + StorageResolution: 1, }; const expected: any = { ...getEmptyPayload() }; expected[expectedKey] = expectedValue; expected._aws.CloudWatchMetrics[0].Metrics.push(expectedMetricDefinition); const context = getContext(); - context.putMetric(expectedKey, expectedValue, expectedUnit,expectedStorageResolution); + context.putMetric(expectedKey, expectedValue, expectedUnit, expectedStorageResolution); // act const resultJson = serializer.serialize(context)[0]; @@ -120,14 +120,14 @@ test('serialize standard resolution metrics', () => { const expectedStorageResolution = StorageResolution.Standard; const expectedMetricDefinition = { Name: expectedKey, - Unit: 'Bits' + Unit: 'Bits', }; const expected: any = { ...getEmptyPayload() }; expected[expectedKey] = expectedValue; expected._aws.CloudWatchMetrics[0].Metrics.push(expectedMetricDefinition); const context = getContext(); - context.putMetric(expectedKey, expectedValue, expectedUnit,expectedStorageResolution); + context.putMetric(expectedKey, expectedValue, expectedUnit, expectedStorageResolution); // act const resultJson = serializer.serialize(context)[0]; diff --git a/src/utils/Validator.ts b/src/utils/Validator.ts index 5e21999..5a7eba1 100644 --- a/src/utils/Validator.ts +++ b/src/utils/Validator.ts @@ -136,7 +136,7 @@ const validateMetric = ( } if (metricNameAndResolutionMap && metricNameAndResolutionMap.has(key)) { - if (metricNameAndResolutionMap.get(key) !== (storageResolution?storageResolution:StorageResolution.Standard)) { + if (metricNameAndResolutionMap.get(key) !== (storageResolution ? storageResolution : StorageResolution.Standard)) { throw new InvalidMetricError( `Resolution for metrics ${key} is already set. A single log event cannot have a metric with two different resolutions.`, ); diff --git a/test/canary/agent/index.js b/test/canary/agent/index.js index 5fdbeb9..6fce43f 100644 --- a/test/canary/agent/index.js +++ b/test/canary/agent/index.js @@ -1,4 +1,4 @@ -const { metricScope, Unit, Configuration, StorageResolution} = require('aws-embedded-metrics'); +const { metricScope, Unit, Configuration, StorageResolution } = require('aws-embedded-metrics'); let version = ''; try { From 6164841d8b0e7700fd2da35272f3b1f072fde9f7 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Tue, 17 Jan 2023 07:12:19 -0800 Subject: [PATCH 10/15] Add unit tests for storage resolution as number --- src/logger/__tests__/MetricsContext.test.ts | 78 ++++++++++++--------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/src/logger/__tests__/MetricsContext.test.ts b/src/logger/__tests__/MetricsContext.test.ts index cdec269..6121786 100644 --- a/src/logger/__tests__/MetricsContext.test.ts +++ b/src/logger/__tests__/MetricsContext.test.ts @@ -351,44 +351,54 @@ test.each([ }); test.each([ - ['', faker.datatype.number(), faker.helpers.arrayElement(Object.values(Unit))], - ['a'.repeat(Constants.MAX_METRIC_NAME_LENGTH + 1), faker.datatype.number(), 'None'], - [faker.random.word(), Number.MAX_VALUE, undefined], - [faker.random.word(), -Number.MAX_VALUE, undefined], - [faker.random.word(), Number.MAX_SAFE_INTEGER + 1, undefined], - [faker.random.word(), -Number.MAX_SAFE_INTEGER - 1, undefined], - [faker.random.word(), parseFloat('not a number'), undefined], - [faker.random.word(), Infinity, faker.helpers.arrayElement(Object.values(Unit))], - [faker.random.word(), -Infinity, faker.helpers.arrayElement(Object.values(Unit))], - [faker.random.word(), faker.datatype.number(), 'Fahrenheit'], - [faker.random.word(), 4, ''], - [faker.random.word(), NaN, faker.helpers.arrayElement(Object.values(Unit))], -])('putMetric with name: %s, value: %d and unit: %s throws error', (metricName, metricValue, metricUnit) => { - // arrange - const context = MetricsContext.empty(); + ['', faker.datatype.number(), faker.helpers.arrayElement(Object.values(Unit)), undefined], + ['a'.repeat(Constants.MAX_METRIC_NAME_LENGTH + 1), faker.datatype.number(), 'None', undefined], + [faker.random.word(), Number.MAX_VALUE, undefined, undefined], + [faker.random.word(), -Number.MAX_VALUE, undefined, undefined], + [faker.random.word(), Number.MAX_SAFE_INTEGER + 1, undefined, undefined], + [faker.random.word(), -Number.MAX_SAFE_INTEGER - 1, undefined, undefined], + [faker.random.word(), parseFloat('not a number'), undefined, undefined], + [faker.random.word(), Infinity, faker.helpers.arrayElement(Object.values(Unit)), undefined], + [faker.random.word(), -Infinity, faker.helpers.arrayElement(Object.values(Unit)), undefined], + [faker.random.word(), faker.datatype.number(), 'Fahrenheit', undefined], + [faker.random.word(), 4, '', undefined], + [faker.random.word(), NaN, faker.helpers.arrayElement(Object.values(Unit)), undefined], + [faker.random.words(3), faker.datatype.number(), Unit.Seconds, 45], + [faker.random.words(3), faker.datatype.number(), Unit.Seconds, 0], +])( + 'putMetric with name: %s, value: %d and unit: %s throws error', + (metricName, metricValue, metricUnit, metricResolution) => { + // arrange + const context = MetricsContext.empty(); - // act - expect(() => { - context.putMetric(metricName, metricValue, metricUnit); - }).toThrow(InvalidMetricError); -}); + // act + expect(() => { + context.putMetric(metricName, metricValue, metricUnit, metricResolution); + }).toThrow(InvalidMetricError); + }, +); test.each([ - [faker.random.word(), faker.datatype.number({ min: -1e3, max: -1 }), undefined], - [faker.random.word(), faker.datatype.number(), faker.helpers.arrayElement(Object.values(Unit))], - [faker.random.words(2), faker.datatype.number(), undefined], - [faker.random.words(3), faker.datatype.number(), Unit.Seconds], - ['Max_Value', Number.MAX_SAFE_INTEGER, Unit.Milliseconds], - ['-Max_Value', -Number.MAX_SAFE_INTEGER, 'Bytes/Second'], -])('putMetric with name: %s, value: %d and unit: %s does not throw error', (metricName, metricValue, metricUnit) => { - // arrange - const context = MetricsContext.empty(); + [faker.random.word(), faker.datatype.number({ min: -1e3, max: -1 }), undefined, undefined], + [faker.random.word(), faker.datatype.number(), faker.helpers.arrayElement(Object.values(Unit)), undefined], + [faker.random.words(2), faker.datatype.number(), undefined, undefined], + [faker.random.words(3), faker.datatype.number(), Unit.Seconds, undefined], + ['Max_Value', Number.MAX_SAFE_INTEGER, Unit.Milliseconds, undefined], + ['-Max_Value', -Number.MAX_SAFE_INTEGER, 'Bytes/Second', undefined], + ['-Max_Value', -Number.MAX_SAFE_INTEGER, 'Bytes/Second', 1], + ['-Max_Value', -Number.MAX_SAFE_INTEGER, 'Bytes/Second', 60], +])( + 'putMetric with name: %s, value: %d and unit: %s does not throw error', + (metricName, metricValue, metricUnit, metricResolution) => { + // arrange + const context = MetricsContext.empty(); - // act - expect(() => { - context.putMetric(metricName, metricValue, metricUnit); - }).not.toThrow(InvalidMetricError); -}); + // act + expect(() => { + context.putMetric(metricName, metricValue, metricUnit, metricResolution); + }).not.toThrow(InvalidMetricError); + }, +); test('put metric with same key and different resolution in single flush throws error', () => { //arrange From db785ca4876e601642abe971101f12a89ce6dff5 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Tue, 17 Jan 2023 07:30:51 -0800 Subject: [PATCH 11/15] Minor fixes --- src/logger/__tests__/MetricsContext.test.ts | 6 +++--- src/serializers/LogSerializer.ts | 4 +--- test/integ/agent/end-to-end.integ.ts | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/logger/__tests__/MetricsContext.test.ts b/src/logger/__tests__/MetricsContext.test.ts index 6121786..69df901 100644 --- a/src/logger/__tests__/MetricsContext.test.ts +++ b/src/logger/__tests__/MetricsContext.test.ts @@ -385,8 +385,8 @@ test.each([ [faker.random.words(3), faker.datatype.number(), Unit.Seconds, undefined], ['Max_Value', Number.MAX_SAFE_INTEGER, Unit.Milliseconds, undefined], ['-Max_Value', -Number.MAX_SAFE_INTEGER, 'Bytes/Second', undefined], - ['-Max_Value', -Number.MAX_SAFE_INTEGER, 'Bytes/Second', 1], - ['-Max_Value', -Number.MAX_SAFE_INTEGER, 'Bytes/Second', 60], + ['-Max_Value', faker.datatype.number(), 'Bytes/Second', 1], + ['-Max_Value', faker.datatype.number(), 'Bytes/Second', 60], ])( 'putMetric with name: %s, value: %d and unit: %s does not throw error', (metricName, metricValue, metricUnit, metricResolution) => { @@ -408,8 +408,8 @@ test('put metric with same key and different resolution in single flush throws e const expectedUnit = 'None'; // act + context.putMetric(expectedKey, expectedValue, expectedUnit, StorageResolution.High); expect(() => { - context.putMetric(expectedKey, expectedValue, expectedUnit, StorageResolution.High); context.putMetric(expectedKey, expectedValue, expectedUnit, StorageResolution.Standard); }).toThrow(InvalidMetricError); }); diff --git a/src/serializers/LogSerializer.ts b/src/serializers/LogSerializer.ts index 1a5149c..e47698d 100644 --- a/src/serializers/LogSerializer.ts +++ b/src/serializers/LogSerializer.ts @@ -112,10 +112,8 @@ export class LogSerializer implements ISerializer { const metricBody: { [key: string]: any } = { Name: metricProgress.name, Unit: metric.unit, + ...(metric.storageResolution == StorageResolution.High ? { StorageResolution: StorageResolution.High } : {}), }; - if (metric.storageResolution === StorageResolution.High) { - metricBody.StorageResolution = StorageResolution.High; - } // eslint-disable-next-line currentBody._aws.CloudWatchMetrics[0].Metrics.push(metricBody); metricProgress.numLeft -= Constants.MAX_VALUES_PER_METRIC; diff --git a/test/integ/agent/end-to-end.integ.ts b/test/integ/agent/end-to-end.integ.ts index feab454..4bba5af 100644 --- a/test/integ/agent/end-to-end.integ.ts +++ b/test/integ/agent/end-to-end.integ.ts @@ -34,7 +34,7 @@ test( const doWork = metricScope(metrics => () => { metrics.putDimensions(dimensions); - metrics.putMetric(metricName, 100, 'Milliseconds', StorageResolution.High); + metrics.putMetric(metricName, 100, 'Milliseconds'); }); // act @@ -58,7 +58,7 @@ test( const doWork = metricScope(metrics => () => { metrics.putDimensions(dimensions); - metrics.putMetric(metricName, 100, 'Milliseconds', StorageResolution.High); + metrics.putMetric(metricName, 100, 'Milliseconds'); metrics.setProperty('RequestId', '422b1569-16f6-4a03-b8f0-fe3fd9b100f8'); }); From 13ddb90e1d90b9f2cc74f17eecd3b60192391385 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Tue, 17 Jan 2023 07:49:26 -0800 Subject: [PATCH 12/15] remove uneccessary null check from metricMap --- src/logger/MetricsContext.ts | 4 +--- src/utils/Validator.ts | 13 +++++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/logger/MetricsContext.ts b/src/logger/MetricsContext.ts index aa5183f..75dcb67 100644 --- a/src/logger/MetricsContext.ts +++ b/src/logger/MetricsContext.ts @@ -196,9 +196,7 @@ export class MetricsContext { } else { this.metrics.set(key, new MetricValues(value, unit, storageResolution)); } - if (this.metricNameAndResolutionMap && !this.metricNameAndResolutionMap.has(key)) { - this.metricNameAndResolutionMap?.set(key, storageResolution || StorageResolution.Standard); - } + this.metricNameAndResolutionMap?.set(key, storageResolution || StorageResolution.Standard); } /** diff --git a/src/utils/Validator.ts b/src/utils/Validator.ts index 5a7eba1..08cd4fc 100644 --- a/src/utils/Validator.ts +++ b/src/utils/Validator.ts @@ -135,12 +135,13 @@ const validateMetric = ( throw new InvalidMetricError(`Metric resolution ${storageResolution} is not valid`); } - if (metricNameAndResolutionMap && metricNameAndResolutionMap.has(key)) { - if (metricNameAndResolutionMap.get(key) !== (storageResolution ? storageResolution : StorageResolution.Standard)) { - throw new InvalidMetricError( - `Resolution for metrics ${key} is already set. A single log event cannot have a metric with two different resolutions.`, - ); - } + if ( + metricNameAndResolutionMap?.has(key) && + metricNameAndResolutionMap.get(key) !== (storageResolution ? storageResolution : StorageResolution.Standard) + ) { + throw new InvalidMetricError( + `Resolution for metrics ${key} is already set. A single log event cannot have a metric with two different resolutions.`, + ); } }; From ce3d4997878624a12e1ae651d127f8485f6b76fc Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Fri, 20 Jan 2023 12:57:41 -0800 Subject: [PATCH 13/15] Minor fix --- src/logger/__tests__/MetricsContext.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/logger/__tests__/MetricsContext.test.ts b/src/logger/__tests__/MetricsContext.test.ts index 69df901..dc76156 100644 --- a/src/logger/__tests__/MetricsContext.test.ts +++ b/src/logger/__tests__/MetricsContext.test.ts @@ -366,7 +366,7 @@ test.each([ [faker.random.words(3), faker.datatype.number(), Unit.Seconds, 45], [faker.random.words(3), faker.datatype.number(), Unit.Seconds, 0], ])( - 'putMetric with name: %s, value: %d and unit: %s throws error', + 'putMetric with name: %s, value: %d, unit: %s and resolution: %d throws error', (metricName, metricValue, metricUnit, metricResolution) => { // arrange const context = MetricsContext.empty(); @@ -388,7 +388,7 @@ test.each([ ['-Max_Value', faker.datatype.number(), 'Bytes/Second', 1], ['-Max_Value', faker.datatype.number(), 'Bytes/Second', 60], ])( - 'putMetric with name: %s, value: %d and unit: %s does not throw error', + 'putMetric with name: %s, value: %d, unit: %s and resolution: %d does not throw error', (metricName, metricValue, metricUnit, metricResolution) => { // arrange const context = MetricsContext.empty(); From ca4aaef3ec0ee3728449cbcd0d3270f1901aded7 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Fri, 20 Jan 2023 14:30:10 -0800 Subject: [PATCH 14/15] Update README.md --- README.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5356129..b9ea7f6 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ const myFunc = metricScope(metrics => async () => { metrics.putDimensions({ Service: "Aggregator" }); metrics.putMetric("ProcessingLatency", 100, Unit.Milliseconds, StorageResolution.Standard); + metrics.putMetric("Memory.HeapUsed", 1600424.0, Unit.Bytes, StorageResolution.High); metrics.setProperty("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8"); // ... }); @@ -60,6 +61,7 @@ const myFunc = metricScope(metrics => async (param1: string, param2: number) => { metrics.putDimensions({ Service: "Aggregator" }); metrics.putMetric("ProcessingLatency", 100, Unit.Milliseconds, StorageResolution.Standard); + metrics.putMetric("Memory.HeapUsed", 1600424.0, Unit.Bytes, StorageResolution.High); metrics.setProperty("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8"); // ... }); @@ -75,7 +77,8 @@ const { createMetricsLogger, Unit, StorageResolution } = require("aws-embedded-m const myFunc = async () => { const metrics = createMetricsLogger(); metrics.putDimensions({ Service: "Aggregator" }); - metrics.putMetric("ProcessingLatency", 100, Unit.Milliseconds), StorageResolution.Standard; + metrics.putMetric("ProcessingLatency", 100, Unit.Milliseconds, StorageResolution.Standard); + metrics.putMetric("Memory.HeapUsed", 1600424.0, Unit.Bytes, StorageResolution.High); metrics.setProperty("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8"); // ... await metrics.flush(); @@ -115,10 +118,18 @@ Requirements: - Values must be in the range of 8.515920e-109 to 1.174271e+108. In addition, special values (for example, NaN, +Infinity, -Infinity) are not supported. - Metrics must meet CloudWatch Metrics requirements, otherwise a `InvalidMetricError` will be thrown. See [MetricDatum](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html) for valid values. +##### Storage Resolution. +An OPTIONAL value representing the storage resolution for the corresponding metric. Setting this to `High` specifies this metric as a high-resolution metric, so that CloudWatch stores the metric with sub-minute resolution down to one second. Setting this to `Standard` specifies this metric as a standard-resolution metric, which CloudWatch stores at 1-minute resolution. If a value is not provided, then a default value of `Standard` is assumed. See [Cloud Watch High-Resolution metrics](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#high-resolution-metrics) + Examples: ```js -putMetric("Latency", 200, Unit.Milliseconds, StorageResolution.Standard) +// Standard Resolution example +putMetric("Latency", 200, Unit.Milliseconds) +putMetric("Latency", 201, Unit.Milliseconds, StorageResolution.Standard) + +// High Resolution example +putMetric("Memory.HeapUsed", 1600424.0, Unit.Bytes, StorageResolution.High); ``` - **setProperty**(String key, Object value) From e83232880bd3bf023ace543df043395bf83202a2 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Fri, 20 Jan 2023 14:31:49 -0800 Subject: [PATCH 15/15] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b9ea7f6..c39085c 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Requirements: - Values must be in the range of 8.515920e-109 to 1.174271e+108. In addition, special values (for example, NaN, +Infinity, -Infinity) are not supported. - Metrics must meet CloudWatch Metrics requirements, otherwise a `InvalidMetricError` will be thrown. See [MetricDatum](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html) for valid values. -##### Storage Resolution. +##### Storage Resolution An OPTIONAL value representing the storage resolution for the corresponding metric. Setting this to `High` specifies this metric as a high-resolution metric, so that CloudWatch stores the metric with sub-minute resolution down to one second. Setting this to `Standard` specifies this metric as a standard-resolution metric, which CloudWatch stores at 1-minute resolution. If a value is not provided, then a default value of `Standard` is assumed. See [Cloud Watch High-Resolution metrics](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#high-resolution-metrics) Examples: