Skip to content

[7.x] Add ML usage collector descriptions to the schema. (#100387) #100860

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pjhampton merged 1 commit into from
May 28, 2021
Merged

[7.x] Add ML usage collector descriptions to the schema. (#100387) #100860

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 158 additions & 29 deletions x-pack/plugins/security_solution/server/usage/collector.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,43 +250,172 @@ export const registerCollector: RegisterCollector = ({
ml_jobs: {
type: 'array',
items: {
job_id: { type: 'keyword' },
open_time: { type: 'keyword' },
create_time: { type: 'keyword' },
finished_time: { type: 'keyword' },
state: { type: 'keyword' },
job_id: {
type: 'keyword',
_meta: { description: 'Identifier for the anomaly detection job' },
},
open_time: {
type: 'keyword',
_meta: {
description: 'For open jobs only, the elapsed time for which the job has been open',
},
},
create_time: {
type: 'keyword',
_meta: { description: 'The time the job was created' },
},
finished_time: {
type: 'keyword',
_meta: {
description: 'If the job closed or failed, this is the time the job finished',
},
},
state: {
type: 'keyword',
_meta: { description: 'The status of the anomaly detection job' },
},
data_counts: {
bucket_count: { type: 'long' },
empty_bucket_count: { type: 'long' },
input_bytes: { type: 'long' },
input_record_count: { type: 'long' },
last_data_time: { type: 'long' },
processed_record_count: { type: 'long' },
bucket_count: {
type: 'long',
_meta: { description: 'The number of buckets processed' },
},
empty_bucket_count: {
type: 'long',
_meta: { description: 'The number of buckets which did not contain any data' },
},
input_bytes: {
type: 'long',
_meta: {
description:
'The number of bytes of input data posted to the anomaly detection job',
},
},
input_record_count: {
type: 'long',
_meta: {
description: 'The number of input documents posted to the anomaly detection job',
},
},
last_data_time: {
type: 'long',
_meta: {
description:
'The timestamp at which data was last analyzed, according to server time',
},
},
processed_record_count: {
type: 'long',
_meta: {
description:
'The number of input documents that have been processed by the anomaly detection job',
},
},
},
model_size_stats: {
bucket_allocation_failures_count: { type: 'long' },
model_bytes: { type: 'long' },
model_bytes_exceeded: { type: 'long' },
model_bytes_memory_limit: { type: 'long' },
peak_model_bytes: { type: 'long' },
bucket_allocation_failures_count: {
type: 'long',
_meta: {
description:
'The number of buckets for which new entities in incoming data were not processed due to insufficient model memory',
},
},
model_bytes: {
type: 'long',
_meta: { description: 'The number of bytes of memory used by the models' },
},
model_bytes_exceeded: {
type: 'long',
_meta: {
description:
'The number of bytes over the high limit for memory usage at the last allocation failure',
},
},
model_bytes_memory_limit: {
type: 'long',
_meta: {
description:
'The upper limit for model memory usage, checked on increasing values',
},
},
peak_model_bytes: {
type: 'long',
_meta: {
description: 'The peak number of bytes of memory ever used by the models',
},
},
},
timing_stats: {
bucket_count: { type: 'long' },
exponential_average_bucket_processing_time_ms: { type: 'long' },
exponential_average_bucket_processing_time_per_hour_ms: { type: 'long' },
maximum_bucket_processing_time_ms: { type: 'long' },
minimum_bucket_processing_time_ms: { type: 'long' },
total_bucket_processing_time_ms: { type: 'long' },
bucket_count: {
type: 'long',
_meta: { description: 'The number of buckets processed' },
},
exponential_average_bucket_processing_time_ms: {
type: 'long',
_meta: {
description:
'Exponential moving average of all bucket processing times, in milliseconds',
},
},
exponential_average_bucket_processing_time_per_hour_ms: {
type: 'long',
_meta: {
description:
'Exponentially-weighted moving average of bucket processing times calculated in a 1 hour time window, in milliseconds',
},
},
maximum_bucket_processing_time_ms: {
type: 'long',
_meta: {
description: 'Maximum among all bucket processing times, in milliseconds',
},
},
minimum_bucket_processing_time_ms: {
type: 'long',
_meta: {
description: 'Minimum among all bucket processing times, in milliseconds',
},
},
total_bucket_processing_time_ms: {
type: 'long',
_meta: { description: 'Sum of all bucket processing times, in milliseconds' },
},
},
datafeed: {
datafeed_id: { type: 'keyword' },
state: { type: 'keyword' },
datafeed_id: {
type: 'keyword',
_meta: {
description: 'A numerical character string that uniquely identifies the datafeed',
},
},
state: {
type: 'keyword',
_meta: { description: 'The status of the datafeed' },
},
timing_stats: {
average_search_time_per_bucket_ms: { type: 'long' },
bucket_count: { type: 'long' },
exponential_average_search_time_per_hour_ms: { type: 'long' },
search_count: { type: 'long' },
total_search_time_ms: { type: 'long' },
average_search_time_per_bucket_ms: {
type: 'long',
_meta: { description: 'The average search time per bucket, in milliseconds' },
},
bucket_count: {
type: 'long',
_meta: { description: 'The number of buckets processed' },
},
exponential_average_search_time_per_hour_ms: {
type: 'long',
_meta: {
description: 'The exponential average search time per hour, in milliseconds',
},
},
search_count: {
type: 'long',
_meta: { description: 'The number of searches run by the datafeed' },
},
total_search_time_ms: {
type: 'long',
_meta: {
description: 'The total time the datafeed spent searching, in milliseconds',
},
},
},
},
},
Expand Down
Loading