Skip to content

[Monitoring] Convert APM-related server files that read from _source to typescript #84829

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Dec 4, 2020
Merged

[Monitoring] Convert APM-related server files that read from _source to typescript #84829

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
df3dd98
get_apms converted
chrisronline Nov 25, 2020
626bfbd
Merge remote-tracking branch 'elastic/master' into monitoring/ecs_ts
chrisronline Nov 30, 2020
a6e7820
More APM ones
chrisronline Nov 30, 2020
65578f8
get_beat_summary
chrisronline Dec 2, 2020
2940f44
Fix test
chrisronline Dec 2, 2020
dcec2e0
This is optional
chrisronline Dec 2, 2020
d32fb96
Merge remote-tracking branch 'elastic/master' into monitoring/source_…
chrisronline Dec 2, 2020
a08fb5f
Fix tests
chrisronline Dec 3, 2020
2c6f8eb
Be more safe
chrisronline Dec 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions ...server/lib/apm/_get_time_of_last_event.js → ...server/lib/apm/_get_time_of_last_event.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { get } from 'lodash';
// @ts-ignore
import { createApmQuery } from './create_apm_query';
// @ts-ignore
import { ApmClusterMetric } from '../metrics';
import { LegacyRequest, ElasticsearchResponse } from '../../types';

export async function getTimeOfLastEvent({
req,
Expand All @@ -15,6 +17,13 @@ export async function getTimeOfLastEvent({
start,
end,
clusterUuid,
}: {
req: LegacyRequest;
callWithRequest: (_req: any, endpoint: string, params: any) => Promise<ElasticsearchResponse>;
apmIndexPattern: string;
start: number;
end: number;
clusterUuid: string;
}) {
const params = {
index: apmIndexPattern,
Expand Down Expand Up @@ -49,5 +58,5 @@ export async function getTimeOfLastEvent({
};

const response = await callWithRequest(req, 'search', params);
return get(response, 'hits.hits[0]._source.timestamp');
return response.hits?.hits.length ? response.hits?.hits[0]._source.timestamp : undefined;
}
70 changes: 47 additions & 23 deletions ...monitoring/server/lib/apm/get_apm_info.js → ...monitoring/server/lib/apm/get_apm_info.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,47 +4,71 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { get, upperFirst } from 'lodash';
import { upperFirst } from 'lodash';
// @ts-ignore
import { checkParam } from '../error_missing_required';
// @ts-ignore
import { createQuery } from '../create_query';
// @ts-ignore
import { getDiffCalculation } from '../beats/_beats_stats';
// @ts-ignore
import { ApmMetric } from '../metrics';
import { getTimeOfLastEvent } from './_get_time_of_last_event';
import { LegacyRequest, ElasticsearchResponse } from '../../types';

export function handleResponse(response, apmUuid) {
const firstStats = get(
response,
'hits.hits[0].inner_hits.first_hit.hits.hits[0]._source.beats_stats'
);
const stats = get(response, 'hits.hits[0]._source.beats_stats');
export function handleResponse(response: ElasticsearchResponse, apmUuid: string) {
if (!response.hits || response.hits.hits.length === 0) {
return {};
}

const eventsTotalFirst = get(firstStats, 'metrics.libbeat.pipeline.events.total', null);
const eventsEmittedFirst = get(firstStats, 'metrics.libbeat.pipeline.events.published', null);
const eventsDroppedFirst = get(firstStats, 'metrics.libbeat.pipeline.events.dropped', null);
const bytesWrittenFirst = get(firstStats, 'metrics.libbeat.output.write.bytes', null);
const firstStats = response.hits.hits[0].inner_hits.first_hit.hits.hits[0]._source.beats_stats;
const stats = response.hits.hits[0]._source.beats_stats;

const eventsTotalLast = get(stats, 'metrics.libbeat.pipeline.events.total', null);
const eventsEmittedLast = get(stats, 'metrics.libbeat.pipeline.events.published', null);
const eventsDroppedLast = get(stats, 'metrics.libbeat.pipeline.events.dropped', null);
const bytesWrittenLast = get(stats, 'metrics.libbeat.output.write.bytes', null);
if (!firstStats || !stats) {
return {};
}

const eventsTotalFirst = firstStats.metrics?.libbeat?.pipeline?.events?.total;
const eventsEmittedFirst = firstStats.metrics?.libbeat?.pipeline?.events?.published;
const eventsDroppedFirst = firstStats.metrics?.libbeat?.pipeline?.events?.dropped;
const bytesWrittenFirst = firstStats.metrics?.libbeat?.output?.write?.bytes;

const eventsTotalLast = stats.metrics?.libbeat?.pipeline?.events?.total;
const eventsEmittedLast = stats.metrics?.libbeat?.pipeline?.events?.published;
const eventsDroppedLast = stats.metrics?.libbeat?.pipeline?.events?.dropped;
const bytesWrittenLast = stats.metrics?.libbeat?.output?.write?.bytes;

return {
uuid: apmUuid,
transportAddress: get(stats, 'beat.host', null),
version: get(stats, 'beat.version', null),
name: get(stats, 'beat.name', null),
type: upperFirst(get(stats, 'beat.type')) || null,
output: upperFirst(get(stats, 'metrics.libbeat.output.type')) || null,
configReloads: get(stats, 'metrics.libbeat.config.reloads', null),
uptime: get(stats, 'metrics.beat.info.uptime.ms', null),
transportAddress: stats.beat?.host,
version: stats.beat?.version,
name: stats.beat?.name,
type: upperFirst(stats.beat?.type) || null,
output: upperFirst(stats.metrics?.libbeat?.output?.type) || null,
configReloads: stats.metrics?.libbeat?.config?.reloads,
uptime: stats.metrics?.beat?.info?.uptime?.ms,
eventsTotal: getDiffCalculation(eventsTotalLast, eventsTotalFirst),
eventsEmitted: getDiffCalculation(eventsEmittedLast, eventsEmittedFirst),
eventsDropped: getDiffCalculation(eventsDroppedLast, eventsDroppedFirst),
bytesWritten: getDiffCalculation(bytesWrittenLast, bytesWrittenFirst),
};
}

export async function getApmInfo(req, apmIndexPattern, { clusterUuid, apmUuid, start, end }) {
export async function getApmInfo(
req: LegacyRequest,
apmIndexPattern: string,
{
clusterUuid,
apmUuid,
start,
end,
}: {
clusterUuid: string;
apmUuid: string;
start: number;
end: number;
}
) {
checkParam(apmIndexPattern, 'apmIndexPattern in beats/getBeatSummary');

const filters = [
Expand Down
63 changes: 37 additions & 26 deletions ...ins/monitoring/server/lib/apm/get_apms.js → ...ins/monitoring/server/lib/apm/get_apms.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,68 +5,79 @@
*/

import moment from 'moment';
import { upperFirst, get } from 'lodash';
import { upperFirst } from 'lodash';
// @ts-ignore
import { checkParam } from '../error_missing_required';
// @ts-ignore
import { createApmQuery } from './create_apm_query';
// @ts-ignore
import { calculateRate } from '../calculate_rate';
// @ts-ignore
import { getDiffCalculation } from './_apm_stats';
import { LegacyRequest, ElasticsearchResponse, ElasticsearchResponseHit } from '../../types';

export function handleResponse(response, start, end) {
const hits = get(response, 'hits.hits', []);
export function handleResponse(response: ElasticsearchResponse, start: number, end: number) {
const initial = { ids: new Set(), beats: [] };
const { beats } = hits.reduce((accum, hit) => {
const stats = get(hit, '_source.beats_stats');
const uuid = get(stats, 'beat.uuid');
const { beats } = response.hits?.hits.reduce((accum: any, hit: ElasticsearchResponseHit) => {
const stats = hit._source.beats_stats;
if (!stats) {
return accum;
}

const earliestStats = hit.inner_hits.earliest.hits.hits[0]._source.beats_stats;
if (!earliestStats) {
return accum;
}

const uuid = stats?.beat?.uuid;

// skip this duplicated beat, newer one was already added
if (accum.ids.has(uuid)) {
return accum;
}

// add another beat summary
accum.ids.add(uuid);
const earliestStats = get(hit, 'inner_hits.earliest.hits.hits[0]._source.beats_stats');

// add the beat
const rateOptions = {
hitTimestamp: get(stats, 'timestamp'),
earliestHitTimestamp: get(earliestStats, 'timestamp'),
hitTimestamp: stats.timestamp,
earliestHitTimestamp: earliestStats.timestamp,
timeWindowMin: start,
timeWindowMax: end,
};

const { rate: bytesSentRate } = calculateRate({
latestTotal: get(stats, 'metrics.libbeat.output.write.bytes'),
earliestTotal: get(earliestStats, 'metrics.libbeat.output.write.bytes'),
latestTotal: stats.metrics?.libbeat?.output?.write?.bytes,
earliestTotal: earliestStats?.metrics?.libbeat?.output?.write?.bytes,
...rateOptions,
});

const { rate: totalEventsRate } = calculateRate({
latestTotal: get(stats, 'metrics.libbeat.pipeline.events.total'),
earliestTotal: get(earliestStats, 'metrics.libbeat.pipeline.events.total'),
latestTotal: stats.metrics?.libbeat?.pipeline?.events?.total,
earliestTotal: earliestStats.metrics?.libbeat?.pipeline?.events?.total,
...rateOptions,
});

const errorsWrittenLatest = get(stats, 'metrics.libbeat.output.write.errors');
const errorsWrittenEarliest = get(earliestStats, 'metrics.libbeat.output.write.errors');
const errorsReadLatest = get(stats, 'metrics.libbeat.output.read.errors');
const errorsReadEarliest = get(earliestStats, 'metrics.libbeat.output.read.errors');
const errorsWrittenLatest = stats.metrics?.libbeat?.output?.write?.errors ?? 0;
const errorsWrittenEarliest = earliestStats.metrics?.libbeat?.output?.write?.errors ?? 0;
const errorsReadLatest = stats.metrics?.libbeat?.output?.read?.errors ?? 0;
const errorsReadEarliest = earliestStats.metrics?.libbeat?.output?.read?.errors ?? 0;
const errors = getDiffCalculation(
errorsWrittenLatest + errorsReadLatest,
errorsWrittenEarliest + errorsReadEarliest
);

accum.beats.push({
uuid: get(stats, 'beat.uuid'),
name: get(stats, 'beat.name'),
type: upperFirst(get(stats, 'beat.type')),
output: upperFirst(get(stats, 'metrics.libbeat.output.type')),
uuid: stats.beat?.uuid,
name: stats.beat?.name,
type: upperFirst(stats.beat?.type),
output: upperFirst(stats.metrics?.libbeat?.output?.type),
total_events_rate: totalEventsRate,
bytes_sent_rate: bytesSentRate,
errors,
memory: get(stats, 'metrics.beat.memstats.memory_alloc'),
version: get(stats, 'beat.version'),
time_of_last_event: get(hit, '_source.timestamp'),
memory: stats.metrics?.beat?.memstats?.memory_alloc,
version: stats.beat?.version,
time_of_last_event: hit._source.timestamp,
});

return accum;
Expand All @@ -75,7 +86,7 @@ export function handleResponse(response, start, end) {
return beats;
}

export async function getApms(req, apmIndexPattern, clusterUuid) {
export async function getApms(req: LegacyRequest, apmIndexPattern: string, clusterUuid: string) {
checkParam(apmIndexPattern, 'apmIndexPattern in getBeats');

const config = req.server.config();
Expand Down
72 changes: 41 additions & 31 deletions ...ring/server/lib/beats/get_beat_summary.js → ...ring/server/lib/beats/get_beat_summary.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,52 +4,62 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { upperFirst, get } from 'lodash';
import { upperFirst } from 'lodash';
import { LegacyRequest, ElasticsearchResponse } from '../../types';
// @ts-ignore
import { checkParam } from '../error_missing_required';
// @ts-ignore
import { createBeatsQuery } from './create_beats_query.js';
// @ts-ignore
import { getDiffCalculation } from './_beats_stats';

export function handleResponse(response, beatUuid) {
const firstStats = get(
response,
'hits.hits[0].inner_hits.first_hit.hits.hits[0]._source.beats_stats'
);
const stats = get(response, 'hits.hits[0]._source.beats_stats');
export function handleResponse(response: ElasticsearchResponse, beatUuid: string) {
if (!response.hits || response.hits.hits.length === 0) {
return {};
}

const eventsTotalFirst = get(firstStats, 'metrics.libbeat.pipeline.events.total', null);
const eventsEmittedFirst = get(firstStats, 'metrics.libbeat.pipeline.events.published', null);
const eventsDroppedFirst = get(firstStats, 'metrics.libbeat.pipeline.events.dropped', null);
const bytesWrittenFirst = get(firstStats, 'metrics.libbeat.output.write.bytes', null);
const firstStats = response.hits.hits[0].inner_hits.first_hit.hits.hits[0]._source.beats_stats;
const stats = response.hits.hits[0]._source.beats_stats;

const eventsTotalLast = get(stats, 'metrics.libbeat.pipeline.events.total', null);
const eventsEmittedLast = get(stats, 'metrics.libbeat.pipeline.events.published', null);
const eventsDroppedLast = get(stats, 'metrics.libbeat.pipeline.events.dropped', null);
const bytesWrittenLast = get(stats, 'metrics.libbeat.output.write.bytes', null);
const handlesHardLimit = get(stats, 'metrics.beat.handles.limit.hard', null);
const handlesSoftLimit = get(stats, 'metrics.beat.handles.limit.soft', null);
const eventsTotalFirst = firstStats?.metrics?.libbeat?.pipeline?.events?.total ?? null;
const eventsEmittedFirst = firstStats?.metrics?.libbeat?.pipeline?.events?.published ?? null;
const eventsDroppedFirst = firstStats?.metrics?.libbeat?.pipeline?.events?.dropped ?? null;
const bytesWrittenFirst = firstStats?.metrics?.libbeat?.output?.write?.bytes ?? null;

const eventsTotalLast = stats?.metrics?.libbeat?.pipeline?.events?.total ?? null;
const eventsEmittedLast = stats?.metrics?.libbeat?.pipeline?.events?.published ?? null;
const eventsDroppedLast = stats?.metrics?.libbeat?.pipeline?.events?.dropped ?? null;
const bytesWrittenLast = stats?.metrics?.libbeat?.output?.write?.bytes ?? null;
const handlesHardLimit = stats?.metrics?.beat?.handles?.limit?.hard ?? null;
const handlesSoftLimit = stats?.metrics?.beat?.handles?.limit?.soft ?? null;

return {
uuid: beatUuid,
transportAddress: get(stats, 'beat.host', null),
version: get(stats, 'beat.version', null),
name: get(stats, 'beat.name', null),
type: upperFirst(get(stats, 'beat.type')) || null,
output: upperFirst(get(stats, 'metrics.libbeat.output.type')) || null,
configReloads: get(stats, 'metrics.libbeat.config.reloads', null),
uptime: get(stats, 'metrics.beat.info.uptime.ms', null),
eventsTotal: getDiffCalculation(eventsTotalLast, eventsTotalFirst),
eventsEmitted: getDiffCalculation(eventsEmittedLast, eventsEmittedFirst),
eventsDropped: getDiffCalculation(eventsDroppedLast, eventsDroppedFirst),
bytesWritten: getDiffCalculation(bytesWrittenLast, bytesWrittenFirst),
transportAddress: stats?.beat?.host ?? null,
version: stats?.beat?.version ?? null,
name: stats?.beat?.name ?? null,
type: upperFirst(stats?.beat?.type) ?? null,
output: upperFirst(stats?.metrics?.libbeat?.output?.type) ?? null,
configReloads: stats?.metrics?.libbeat?.config?.reloads ?? null,
uptime: stats?.metrics?.beat?.info?.uptime?.ms ?? null,
eventsTotal: getDiffCalculation(eventsTotalLast, eventsTotalFirst) ?? null,
eventsEmitted: getDiffCalculation(eventsEmittedLast, eventsEmittedFirst) ?? null,
eventsDropped: getDiffCalculation(eventsDroppedLast, eventsDroppedFirst) ?? null,
bytesWritten: getDiffCalculation(bytesWrittenLast, bytesWrittenFirst) ?? null,
handlesHardLimit,
handlesSoftLimit,
};
}

export async function getBeatSummary(
req,
beatsIndexPattern,
{ clusterUuid, beatUuid, start, end }
req: LegacyRequest,
beatsIndexPattern: string,
{
clusterUuid,
beatUuid,
start,
end,
}: { clusterUuid: string; beatUuid: string; start: number; end: number }
) {
checkParam(beatsIndexPattern, 'beatsIndexPattern in beats/getBeatSummary');

Expand Down
Loading