diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 652626d16df49..b8745359bb61a 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -63111,6 +63111,228 @@ paths: - action - from - to + - allOf: + - description: Base processor options plus conditional execution + type: object + properties: + action: + enum: + - network_direction + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + destination_ip: + description: A non-empty string. + minLength: 1 + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + ignore_missing: + type: boolean + source_ip: + description: A non-empty string. + minLength: 1 + type: string + target_field: + description: A non-empty string. + minLength: 1 + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than comparison value. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than-or-equal comparison value. + includes: + anyOf: + - type: string + - type: number + - type: boolean + description: Checks if multivalue field includes the value. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than comparison value. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than-or-equal comparison value. + neq: + anyOf: + - type: string + - type: number + - type: boolean + description: Inequality comparison value. + range: + additionalProperties: false + description: Range comparison values. + type: object + properties: + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and + - additionalProperties: false + description: A logical OR that groups multiple conditions. + type: object + properties: + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array + required: + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - source_ip + - destination_ip + - anyOf: + - additionalProperties: false + type: object + properties: + internal_networks: + items: + type: string + type: array + required: + - internal_networks + - additionalProperties: false + type: object + properties: + internal_networks_field: + description: A non-empty string. + minLength: 1 + type: string + required: + - internal_networks_field - additionalProperties: false description: Manual ingest pipeline wrapper around native Elasticsearch processors type: object @@ -67827,161 +68049,383 @@ paths: - action - from - to - - additionalProperties: false - description: Manual ingest pipeline wrapper around native Elasticsearch processors - type: object - properties: - action: - description: Manual ingest pipeline - executes raw Elasticsearch ingest processors - enum: - - manual_ingest_pipeline - type: string - customIdentifier: - description: Custom identifier to correlate this processor across outputs - minLength: 1 - type: string - description: - description: Human-readable notes about this processor step - type: string - ignore_failure: - description: Continue pipeline execution if this processor fails - type: boolean - on_failure: - description: Fallback processors to run when a processor fails - items: - additionalProperties: {} - type: object - type: array - processors: - description: List of raw Elasticsearch ingest processors to run - items: - additionalProperties: false - type: object - properties: - append: {} - attachment: {} - bytes: {} - circle: {} - community_id: {} - convert: {} - csv: {} - date: {} - date_index_name: {} - dissect: {} - dot_expander: {} - drop: {} - enrich: {} - fail: {} - fingerprint: {} - foreach: {} - geo_grid: {} - geoip: {} - grok: {} - gsub: {} - html_strip: {} - inference: {} - ip_location: {} - join: {} - json: {} - kv: {} - lowercase: {} - network_direction: {} - pipeline: {} - redact: {} - registered_domain: {} - remove: {} - rename: {} - reroute: {} - script: {} - set: {} - set_security_user: {} - sort: {} - split: {} - terminate: {} - trim: {} - uppercase: {} - uri_parts: {} - urldecode: {} - user_agent: {} - required: - - append - - attachment - - bytes - - circle - - community_id - - convert - - csv - - date - - date_index_name - - dissect - - dot_expander - - drop - - enrich - - fail - - fingerprint - - foreach - - ip_location - - geo_grid - - geoip - - grok - - gsub - - html_strip - - inference - - join - - json - - kv - - lowercase - - network_direction - - pipeline - - redact - - registered_domain - - remove - - rename - - reroute - - script - - set - - set_security_user - - sort - - split - - terminate - - trim - - uppercase - - urldecode - - uri_parts - - user_agent - type: array - tag: - description: Optional ingest processor tag for Elasticsearch - type: string - where: - anyOf: - - anyOf: - - additionalProperties: false - description: A condition that compares a field to a value or range using an operator as the key. - type: object - properties: - contains: - anyOf: - - type: string - - type: number - - type: boolean - description: Contains comparison value. - endsWith: - anyOf: - - type: string - - type: number - - type: boolean - description: Ends-with comparison value. - eq: - anyOf: - - type: string - - type: number - - type: boolean - description: Equality comparison value. - field: - description: The document field to filter on. - minLength: 1 - type: string + - allOf: + - description: Base processor options plus conditional execution + type: object + properties: + action: + enum: + - network_direction + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + destination_ip: + description: A non-empty string. + minLength: 1 + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + ignore_missing: + type: boolean + source_ip: + description: A non-empty string. + minLength: 1 + type: string + target_field: + description: A non-empty string. + minLength: 1 + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than comparison value. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than-or-equal comparison value. + includes: + anyOf: + - type: string + - type: number + - type: boolean + description: Checks if multivalue field includes the value. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than comparison value. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than-or-equal comparison value. + neq: + anyOf: + - type: string + - type: number + - type: boolean + description: Inequality comparison value. + range: + additionalProperties: false + description: Range comparison values. + type: object + properties: + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and + - additionalProperties: false + description: A logical OR that groups multiple conditions. + type: object + properties: + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array + required: + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - source_ip + - destination_ip + - anyOf: + - additionalProperties: false + type: object + properties: + internal_networks: + items: + type: string + type: array + required: + - internal_networks + - additionalProperties: false + type: object + properties: + internal_networks_field: + description: A non-empty string. + minLength: 1 + type: string + required: + - internal_networks_field + - additionalProperties: false + description: Manual ingest pipeline wrapper around native Elasticsearch processors + type: object + properties: + action: + description: Manual ingest pipeline - executes raw Elasticsearch ingest processors + enum: + - manual_ingest_pipeline + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + on_failure: + description: Fallback processors to run when a processor fails + items: + additionalProperties: {} + type: object + type: array + processors: + description: List of raw Elasticsearch ingest processors to run + items: + additionalProperties: false + type: object + properties: + append: {} + attachment: {} + bytes: {} + circle: {} + community_id: {} + convert: {} + csv: {} + date: {} + date_index_name: {} + dissect: {} + dot_expander: {} + drop: {} + enrich: {} + fail: {} + fingerprint: {} + foreach: {} + geo_grid: {} + geoip: {} + grok: {} + gsub: {} + html_strip: {} + inference: {} + ip_location: {} + join: {} + json: {} + kv: {} + lowercase: {} + network_direction: {} + pipeline: {} + redact: {} + registered_domain: {} + remove: {} + rename: {} + reroute: {} + script: {} + set: {} + set_security_user: {} + sort: {} + split: {} + terminate: {} + trim: {} + uppercase: {} + uri_parts: {} + urldecode: {} + user_agent: {} + required: + - append + - attachment + - bytes + - circle + - community_id + - convert + - csv + - date + - date_index_name + - dissect + - dot_expander + - drop + - enrich + - fail + - fingerprint + - foreach + - ip_location + - geo_grid + - geoip + - grok + - gsub + - html_strip + - inference + - join + - json + - kv + - lowercase + - network_direction + - pipeline + - redact + - registered_domain + - remove + - rename + - reroute + - script + - set + - set_security_user + - sort + - split + - terminate + - trim + - uppercase + - urldecode + - uri_parts + - user_agent + type: array + tag: + description: Optional ingest processor tag for Elasticsearch + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string gt: anyOf: - type: string @@ -72258,32 +72702,254 @@ paths: required: - not - additionalProperties: false - description: A condition that always evaluates to false. + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - from + - to + - allOf: + - description: Base processor options plus conditional execution + type: object + properties: + action: + enum: + - network_direction + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + destination_ip: + description: A non-empty string. + minLength: 1 + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + ignore_missing: + type: boolean + source_ip: + description: A non-empty string. + minLength: 1 + type: string + target_field: + description: A non-empty string. + minLength: 1 + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than comparison value. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than-or-equal comparison value. + includes: + anyOf: + - type: string + - type: number + - type: boolean + description: Checks if multivalue field includes the value. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than comparison value. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than-or-equal comparison value. + neq: + anyOf: + - type: string + - type: number + - type: boolean + description: Inequality comparison value. + range: + additionalProperties: false + description: Range comparison values. + type: object + properties: + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and + - additionalProperties: false + description: A logical OR that groups multiple conditions. + type: object + properties: + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array + required: + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - source_ip + - destination_ip + - anyOf: + - additionalProperties: false type: object properties: - never: - additionalProperties: false - description: An empty object. This condition never matches. - type: object - properties: {} + internal_networks: + items: + type: string + type: array required: - - never + - internal_networks - additionalProperties: false - description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. type: object properties: - always: - additionalProperties: false - description: An empty object. This condition always matches. - type: object - properties: {} + internal_networks_field: + description: A non-empty string. + minLength: 1 + type: string required: - - always - description: Conditional expression controlling whether this processor runs - required: - - action - - from - - to + - internal_networks_field - additionalProperties: false description: Manual ingest pipeline wrapper around native Elasticsearch processors type: object @@ -76742,76 +77408,298 @@ paths: - type: boolean description: Starts-with comparison value. required: - - field + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and + - additionalProperties: false + description: A logical OR that groups multiple conditions. + type: object + properties: + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array + required: + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - from + - to + - allOf: + - description: Base processor options plus conditional execution + type: object + properties: + action: + enum: + - network_direction + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + destination_ip: + description: A non-empty string. + minLength: 1 + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + ignore_missing: + type: boolean + source_ip: + description: A non-empty string. + minLength: 1 + type: string + target_field: + description: A non-empty string. + minLength: 1 + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than comparison value. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than-or-equal comparison value. + includes: + anyOf: + - type: string + - type: number + - type: boolean + description: Checks if multivalue field includes the value. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than comparison value. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than-or-equal comparison value. + neq: + anyOf: + - type: string + - type: number + - type: boolean + description: Inequality comparison value. + range: + additionalProperties: false + description: Range comparison values. + type: object + properties: + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and - additionalProperties: false - description: A condition that checks for the existence or non-existence of a field. + description: A logical OR that groups multiple conditions. type: object properties: - exists: - description: Indicates whether the field exists or not. - type: boolean - field: - description: The document field to check. - minLength: 1 - type: string + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array required: - - field - description: A basic filter condition, either unary or binary. - - additionalProperties: false - description: A logical AND that groups multiple conditions. - type: object - properties: - and: - description: An array of conditions. All sub-conditions must be true for this condition to be true. - items: {} - type: array - required: - - and + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - source_ip + - destination_ip + - anyOf: - additionalProperties: false - description: A logical OR that groups multiple conditions. type: object properties: - or: - description: An array of conditions. At least one sub-condition must be true for this condition to be true. - items: {} + internal_networks: + items: + type: string type: array required: - - or - - additionalProperties: false - description: A logical NOT that negates a condition. - type: object - properties: - not: - description: A condition that negates another condition. - required: - - not - - additionalProperties: false - description: A condition that always evaluates to false. - type: object - properties: - never: - additionalProperties: false - description: An empty object. This condition never matches. - type: object - properties: {} - required: - - never + - internal_networks - additionalProperties: false - description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. type: object properties: - always: - additionalProperties: false - description: An empty object. This condition always matches. - type: object - properties: {} + internal_networks_field: + description: A non-empty string. + minLength: 1 + type: string required: - - always - description: Conditional expression controlling whether this processor runs - required: - - action - - from - - to + - internal_networks_field - additionalProperties: false description: Manual ingest pipeline wrapper around native Elasticsearch processors type: object @@ -81525,84 +82413,306 @@ paths: - type: string - type: number - type: boolean - description: A value that can be a string, number, or boolean. - startsWith: - anyOf: - - type: string - - type: number - - type: boolean - description: Starts-with comparison value. + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and + - additionalProperties: false + description: A logical OR that groups multiple conditions. + type: object + properties: + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array + required: + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - from + - to + - allOf: + - description: Base processor options plus conditional execution + type: object + properties: + action: + enum: + - network_direction + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + destination_ip: + description: A non-empty string. + minLength: 1 + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + ignore_missing: + type: boolean + source_ip: + description: A non-empty string. + minLength: 1 + type: string + target_field: + description: A non-empty string. + minLength: 1 + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than comparison value. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than-or-equal comparison value. + includes: + anyOf: + - type: string + - type: number + - type: boolean + description: Checks if multivalue field includes the value. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than comparison value. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than-or-equal comparison value. + neq: + anyOf: + - type: string + - type: number + - type: boolean + description: Inequality comparison value. + range: + additionalProperties: false + description: Range comparison values. + type: object + properties: + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array required: - - field + - and - additionalProperties: false - description: A condition that checks for the existence or non-existence of a field. + description: A logical OR that groups multiple conditions. type: object properties: - exists: - description: Indicates whether the field exists or not. - type: boolean - field: - description: The document field to check. - minLength: 1 - type: string + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array required: - - field - description: A basic filter condition, either unary or binary. - - additionalProperties: false - description: A logical AND that groups multiple conditions. - type: object - properties: - and: - description: An array of conditions. All sub-conditions must be true for this condition to be true. - items: {} - type: array - required: - - and + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - source_ip + - destination_ip + - anyOf: - additionalProperties: false - description: A logical OR that groups multiple conditions. type: object properties: - or: - description: An array of conditions. At least one sub-condition must be true for this condition to be true. - items: {} + internal_networks: + items: + type: string type: array required: - - or - - additionalProperties: false - description: A logical NOT that negates a condition. - type: object - properties: - not: - description: A condition that negates another condition. - required: - - not - - additionalProperties: false - description: A condition that always evaluates to false. - type: object - properties: - never: - additionalProperties: false - description: An empty object. This condition never matches. - type: object - properties: {} - required: - - never + - internal_networks - additionalProperties: false - description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. type: object properties: - always: - additionalProperties: false - description: An empty object. This condition always matches. - type: object - properties: {} + internal_networks_field: + description: A non-empty string. + minLength: 1 + type: string required: - - always - description: Conditional expression controlling whether this processor runs - required: - - action - - from - - to + - internal_networks_field - additionalProperties: false description: Manual ingest pipeline wrapper around native Elasticsearch processors type: object @@ -86003,6 +87113,228 @@ paths: - action - from - to + - allOf: + - description: Base processor options plus conditional execution + type: object + properties: + action: + enum: + - network_direction + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + destination_ip: + description: A non-empty string. + minLength: 1 + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + ignore_missing: + type: boolean + source_ip: + description: A non-empty string. + minLength: 1 + type: string + target_field: + description: A non-empty string. + minLength: 1 + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than comparison value. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than-or-equal comparison value. + includes: + anyOf: + - type: string + - type: number + - type: boolean + description: Checks if multivalue field includes the value. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than comparison value. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than-or-equal comparison value. + neq: + anyOf: + - type: string + - type: number + - type: boolean + description: Inequality comparison value. + range: + additionalProperties: false + description: Range comparison values. + type: object + properties: + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and + - additionalProperties: false + description: A logical OR that groups multiple conditions. + type: object + properties: + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array + required: + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - source_ip + - destination_ip + - anyOf: + - additionalProperties: false + type: object + properties: + internal_networks: + items: + type: string + type: array + required: + - internal_networks + - additionalProperties: false + type: object + properties: + internal_networks_field: + description: A non-empty string. + minLength: 1 + type: string + required: + - internal_networks_field - additionalProperties: false description: Manual ingest pipeline wrapper around native Elasticsearch processors type: object diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index 7a8a5c9f57e2d..60dcba6ee1fb4 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -67565,6 +67565,228 @@ paths: - action - from - to + - allOf: + - description: Base processor options plus conditional execution + type: object + properties: + action: + enum: + - network_direction + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + destination_ip: + description: A non-empty string. + minLength: 1 + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + ignore_missing: + type: boolean + source_ip: + description: A non-empty string. + minLength: 1 + type: string + target_field: + description: A non-empty string. + minLength: 1 + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than comparison value. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than-or-equal comparison value. + includes: + anyOf: + - type: string + - type: number + - type: boolean + description: Checks if multivalue field includes the value. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than comparison value. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than-or-equal comparison value. + neq: + anyOf: + - type: string + - type: number + - type: boolean + description: Inequality comparison value. + range: + additionalProperties: false + description: Range comparison values. + type: object + properties: + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and + - additionalProperties: false + description: A logical OR that groups multiple conditions. + type: object + properties: + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array + required: + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - source_ip + - destination_ip + - anyOf: + - additionalProperties: false + type: object + properties: + internal_networks: + items: + type: string + type: array + required: + - internal_networks + - additionalProperties: false + type: object + properties: + internal_networks_field: + description: A non-empty string. + minLength: 1 + type: string + required: + - internal_networks_field - additionalProperties: false description: Manual ingest pipeline wrapper around native Elasticsearch processors type: object @@ -72281,161 +72503,383 @@ paths: - action - from - to - - additionalProperties: false - description: Manual ingest pipeline wrapper around native Elasticsearch processors - type: object - properties: - action: - description: Manual ingest pipeline - executes raw Elasticsearch ingest processors - enum: - - manual_ingest_pipeline - type: string - customIdentifier: - description: Custom identifier to correlate this processor across outputs - minLength: 1 - type: string - description: - description: Human-readable notes about this processor step - type: string - ignore_failure: - description: Continue pipeline execution if this processor fails - type: boolean - on_failure: - description: Fallback processors to run when a processor fails - items: - additionalProperties: {} - type: object - type: array - processors: - description: List of raw Elasticsearch ingest processors to run - items: - additionalProperties: false - type: object - properties: - append: {} - attachment: {} - bytes: {} - circle: {} - community_id: {} - convert: {} - csv: {} - date: {} - date_index_name: {} - dissect: {} - dot_expander: {} - drop: {} - enrich: {} - fail: {} - fingerprint: {} - foreach: {} - geo_grid: {} - geoip: {} - grok: {} - gsub: {} - html_strip: {} - inference: {} - ip_location: {} - join: {} - json: {} - kv: {} - lowercase: {} - network_direction: {} - pipeline: {} - redact: {} - registered_domain: {} - remove: {} - rename: {} - reroute: {} - script: {} - set: {} - set_security_user: {} - sort: {} - split: {} - terminate: {} - trim: {} - uppercase: {} - uri_parts: {} - urldecode: {} - user_agent: {} - required: - - append - - attachment - - bytes - - circle - - community_id - - convert - - csv - - date - - date_index_name - - dissect - - dot_expander - - drop - - enrich - - fail - - fingerprint - - foreach - - ip_location - - geo_grid - - geoip - - grok - - gsub - - html_strip - - inference - - join - - json - - kv - - lowercase - - network_direction - - pipeline - - redact - - registered_domain - - remove - - rename - - reroute - - script - - set - - set_security_user - - sort - - split - - terminate - - trim - - uppercase - - urldecode - - uri_parts - - user_agent - type: array - tag: - description: Optional ingest processor tag for Elasticsearch - type: string - where: - anyOf: - - anyOf: - - additionalProperties: false - description: A condition that compares a field to a value or range using an operator as the key. - type: object - properties: - contains: - anyOf: - - type: string - - type: number - - type: boolean - description: Contains comparison value. - endsWith: - anyOf: - - type: string - - type: number - - type: boolean - description: Ends-with comparison value. - eq: - anyOf: - - type: string - - type: number - - type: boolean - description: Equality comparison value. - field: - description: The document field to filter on. - minLength: 1 - type: string + - allOf: + - description: Base processor options plus conditional execution + type: object + properties: + action: + enum: + - network_direction + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + destination_ip: + description: A non-empty string. + minLength: 1 + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + ignore_missing: + type: boolean + source_ip: + description: A non-empty string. + minLength: 1 + type: string + target_field: + description: A non-empty string. + minLength: 1 + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than comparison value. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than-or-equal comparison value. + includes: + anyOf: + - type: string + - type: number + - type: boolean + description: Checks if multivalue field includes the value. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than comparison value. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than-or-equal comparison value. + neq: + anyOf: + - type: string + - type: number + - type: boolean + description: Inequality comparison value. + range: + additionalProperties: false + description: Range comparison values. + type: object + properties: + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and + - additionalProperties: false + description: A logical OR that groups multiple conditions. + type: object + properties: + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array + required: + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - source_ip + - destination_ip + - anyOf: + - additionalProperties: false + type: object + properties: + internal_networks: + items: + type: string + type: array + required: + - internal_networks + - additionalProperties: false + type: object + properties: + internal_networks_field: + description: A non-empty string. + minLength: 1 + type: string + required: + - internal_networks_field + - additionalProperties: false + description: Manual ingest pipeline wrapper around native Elasticsearch processors + type: object + properties: + action: + description: Manual ingest pipeline - executes raw Elasticsearch ingest processors + enum: + - manual_ingest_pipeline + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + on_failure: + description: Fallback processors to run when a processor fails + items: + additionalProperties: {} + type: object + type: array + processors: + description: List of raw Elasticsearch ingest processors to run + items: + additionalProperties: false + type: object + properties: + append: {} + attachment: {} + bytes: {} + circle: {} + community_id: {} + convert: {} + csv: {} + date: {} + date_index_name: {} + dissect: {} + dot_expander: {} + drop: {} + enrich: {} + fail: {} + fingerprint: {} + foreach: {} + geo_grid: {} + geoip: {} + grok: {} + gsub: {} + html_strip: {} + inference: {} + ip_location: {} + join: {} + json: {} + kv: {} + lowercase: {} + network_direction: {} + pipeline: {} + redact: {} + registered_domain: {} + remove: {} + rename: {} + reroute: {} + script: {} + set: {} + set_security_user: {} + sort: {} + split: {} + terminate: {} + trim: {} + uppercase: {} + uri_parts: {} + urldecode: {} + user_agent: {} + required: + - append + - attachment + - bytes + - circle + - community_id + - convert + - csv + - date + - date_index_name + - dissect + - dot_expander + - drop + - enrich + - fail + - fingerprint + - foreach + - ip_location + - geo_grid + - geoip + - grok + - gsub + - html_strip + - inference + - join + - json + - kv + - lowercase + - network_direction + - pipeline + - redact + - registered_domain + - remove + - rename + - reroute + - script + - set + - set_security_user + - sort + - split + - terminate + - trim + - uppercase + - urldecode + - uri_parts + - user_agent + type: array + tag: + description: Optional ingest processor tag for Elasticsearch + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string gt: anyOf: - type: string @@ -76712,32 +77156,254 @@ paths: required: - not - additionalProperties: false - description: A condition that always evaluates to false. + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - from + - to + - allOf: + - description: Base processor options plus conditional execution + type: object + properties: + action: + enum: + - network_direction + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + destination_ip: + description: A non-empty string. + minLength: 1 + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + ignore_missing: + type: boolean + source_ip: + description: A non-empty string. + minLength: 1 + type: string + target_field: + description: A non-empty string. + minLength: 1 + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than comparison value. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than-or-equal comparison value. + includes: + anyOf: + - type: string + - type: number + - type: boolean + description: Checks if multivalue field includes the value. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than comparison value. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than-or-equal comparison value. + neq: + anyOf: + - type: string + - type: number + - type: boolean + description: Inequality comparison value. + range: + additionalProperties: false + description: Range comparison values. + type: object + properties: + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and + - additionalProperties: false + description: A logical OR that groups multiple conditions. + type: object + properties: + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array + required: + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - source_ip + - destination_ip + - anyOf: + - additionalProperties: false type: object properties: - never: - additionalProperties: false - description: An empty object. This condition never matches. - type: object - properties: {} + internal_networks: + items: + type: string + type: array required: - - never + - internal_networks - additionalProperties: false - description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. type: object properties: - always: - additionalProperties: false - description: An empty object. This condition always matches. - type: object - properties: {} + internal_networks_field: + description: A non-empty string. + minLength: 1 + type: string required: - - always - description: Conditional expression controlling whether this processor runs - required: - - action - - from - - to + - internal_networks_field - additionalProperties: false description: Manual ingest pipeline wrapper around native Elasticsearch processors type: object @@ -81196,76 +81862,298 @@ paths: - type: boolean description: Starts-with comparison value. required: - - field + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and + - additionalProperties: false + description: A logical OR that groups multiple conditions. + type: object + properties: + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array + required: + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - from + - to + - allOf: + - description: Base processor options plus conditional execution + type: object + properties: + action: + enum: + - network_direction + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + destination_ip: + description: A non-empty string. + minLength: 1 + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + ignore_missing: + type: boolean + source_ip: + description: A non-empty string. + minLength: 1 + type: string + target_field: + description: A non-empty string. + minLength: 1 + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than comparison value. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than-or-equal comparison value. + includes: + anyOf: + - type: string + - type: number + - type: boolean + description: Checks if multivalue field includes the value. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than comparison value. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than-or-equal comparison value. + neq: + anyOf: + - type: string + - type: number + - type: boolean + description: Inequality comparison value. + range: + additionalProperties: false + description: Range comparison values. + type: object + properties: + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and - additionalProperties: false - description: A condition that checks for the existence or non-existence of a field. + description: A logical OR that groups multiple conditions. type: object properties: - exists: - description: Indicates whether the field exists or not. - type: boolean - field: - description: The document field to check. - minLength: 1 - type: string + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array required: - - field - description: A basic filter condition, either unary or binary. - - additionalProperties: false - description: A logical AND that groups multiple conditions. - type: object - properties: - and: - description: An array of conditions. All sub-conditions must be true for this condition to be true. - items: {} - type: array - required: - - and + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - source_ip + - destination_ip + - anyOf: - additionalProperties: false - description: A logical OR that groups multiple conditions. type: object properties: - or: - description: An array of conditions. At least one sub-condition must be true for this condition to be true. - items: {} + internal_networks: + items: + type: string type: array required: - - or - - additionalProperties: false - description: A logical NOT that negates a condition. - type: object - properties: - not: - description: A condition that negates another condition. - required: - - not - - additionalProperties: false - description: A condition that always evaluates to false. - type: object - properties: - never: - additionalProperties: false - description: An empty object. This condition never matches. - type: object - properties: {} - required: - - never + - internal_networks - additionalProperties: false - description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. type: object properties: - always: - additionalProperties: false - description: An empty object. This condition always matches. - type: object - properties: {} + internal_networks_field: + description: A non-empty string. + minLength: 1 + type: string required: - - always - description: Conditional expression controlling whether this processor runs - required: - - action - - from - - to + - internal_networks_field - additionalProperties: false description: Manual ingest pipeline wrapper around native Elasticsearch processors type: object @@ -85979,84 +86867,306 @@ paths: - type: string - type: number - type: boolean - description: A value that can be a string, number, or boolean. - startsWith: - anyOf: - - type: string - - type: number - - type: boolean - description: Starts-with comparison value. + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and + - additionalProperties: false + description: A logical OR that groups multiple conditions. + type: object + properties: + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array + required: + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - from + - to + - allOf: + - description: Base processor options plus conditional execution + type: object + properties: + action: + enum: + - network_direction + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + destination_ip: + description: A non-empty string. + minLength: 1 + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + ignore_missing: + type: boolean + source_ip: + description: A non-empty string. + minLength: 1 + type: string + target_field: + description: A non-empty string. + minLength: 1 + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than comparison value. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than-or-equal comparison value. + includes: + anyOf: + - type: string + - type: number + - type: boolean + description: Checks if multivalue field includes the value. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than comparison value. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than-or-equal comparison value. + neq: + anyOf: + - type: string + - type: number + - type: boolean + description: Inequality comparison value. + range: + additionalProperties: false + description: Range comparison values. + type: object + properties: + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array required: - - field + - and - additionalProperties: false - description: A condition that checks for the existence or non-existence of a field. + description: A logical OR that groups multiple conditions. type: object properties: - exists: - description: Indicates whether the field exists or not. - type: boolean - field: - description: The document field to check. - minLength: 1 - type: string + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array required: - - field - description: A basic filter condition, either unary or binary. - - additionalProperties: false - description: A logical AND that groups multiple conditions. - type: object - properties: - and: - description: An array of conditions. All sub-conditions must be true for this condition to be true. - items: {} - type: array - required: - - and + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - source_ip + - destination_ip + - anyOf: - additionalProperties: false - description: A logical OR that groups multiple conditions. type: object properties: - or: - description: An array of conditions. At least one sub-condition must be true for this condition to be true. - items: {} + internal_networks: + items: + type: string type: array required: - - or - - additionalProperties: false - description: A logical NOT that negates a condition. - type: object - properties: - not: - description: A condition that negates another condition. - required: - - not - - additionalProperties: false - description: A condition that always evaluates to false. - type: object - properties: - never: - additionalProperties: false - description: An empty object. This condition never matches. - type: object - properties: {} - required: - - never + - internal_networks - additionalProperties: false - description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. type: object properties: - always: - additionalProperties: false - description: An empty object. This condition always matches. - type: object - properties: {} + internal_networks_field: + description: A non-empty string. + minLength: 1 + type: string required: - - always - description: Conditional expression controlling whether this processor runs - required: - - action - - from - - to + - internal_networks_field - additionalProperties: false description: Manual ingest pipeline wrapper around native Elasticsearch processors type: object @@ -90457,6 +91567,228 @@ paths: - action - from - to + - allOf: + - description: Base processor options plus conditional execution + type: object + properties: + action: + enum: + - network_direction + type: string + customIdentifier: + description: Custom identifier to correlate this processor across outputs + minLength: 1 + type: string + description: + description: Human-readable notes about this processor step + type: string + destination_ip: + description: A non-empty string. + minLength: 1 + type: string + ignore_failure: + description: Continue pipeline execution if this processor fails + type: boolean + ignore_missing: + type: boolean + source_ip: + description: A non-empty string. + minLength: 1 + type: string + target_field: + description: A non-empty string. + minLength: 1 + type: string + where: + anyOf: + - anyOf: + - additionalProperties: false + description: A condition that compares a field to a value or range using an operator as the key. + type: object + properties: + contains: + anyOf: + - type: string + - type: number + - type: boolean + description: Contains comparison value. + endsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Ends-with comparison value. + eq: + anyOf: + - type: string + - type: number + - type: boolean + description: Equality comparison value. + field: + description: The document field to filter on. + minLength: 1 + type: string + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than comparison value. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: Greater-than-or-equal comparison value. + includes: + anyOf: + - type: string + - type: number + - type: boolean + description: Checks if multivalue field includes the value. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than comparison value. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: Less-than-or-equal comparison value. + neq: + anyOf: + - type: string + - type: number + - type: boolean + description: Inequality comparison value. + range: + additionalProperties: false + description: Range comparison values. + type: object + properties: + gt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + gte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lt: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + lte: + anyOf: + - type: string + - type: number + - type: boolean + description: A value that can be a string, number, or boolean. + startsWith: + anyOf: + - type: string + - type: number + - type: boolean + description: Starts-with comparison value. + required: + - field + - additionalProperties: false + description: A condition that checks for the existence or non-existence of a field. + type: object + properties: + exists: + description: Indicates whether the field exists or not. + type: boolean + field: + description: The document field to check. + minLength: 1 + type: string + required: + - field + description: A basic filter condition, either unary or binary. + - additionalProperties: false + description: A logical AND that groups multiple conditions. + type: object + properties: + and: + description: An array of conditions. All sub-conditions must be true for this condition to be true. + items: {} + type: array + required: + - and + - additionalProperties: false + description: A logical OR that groups multiple conditions. + type: object + properties: + or: + description: An array of conditions. At least one sub-condition must be true for this condition to be true. + items: {} + type: array + required: + - or + - additionalProperties: false + description: A logical NOT that negates a condition. + type: object + properties: + not: + description: A condition that negates another condition. + required: + - not + - additionalProperties: false + description: A condition that always evaluates to false. + type: object + properties: + never: + additionalProperties: false + description: An empty object. This condition never matches. + type: object + properties: {} + required: + - never + - additionalProperties: false + description: A condition that always evaluates to true. Useful for catch-all scenarios, but use with caution as partitions are ordered. + type: object + properties: + always: + additionalProperties: false + description: An empty object. This condition always matches. + type: object + properties: {} + required: + - always + description: Conditional expression controlling whether this processor runs + required: + - action + - source_ip + - destination_ip + - anyOf: + - additionalProperties: false + type: object + properties: + internal_networks: + items: + type: string + type: array + required: + - internal_networks + - additionalProperties: false + type: object + properties: + internal_networks_field: + description: A non-empty string. + minLength: 1 + type: string + required: + - internal_networks_field - additionalProperties: false description: Manual ingest pipeline wrapper around native Elasticsearch processors type: object diff --git a/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/cross_compatibility/network_direction.spec.ts b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/cross_compatibility/network_direction.spec.ts new file mode 100644 index 0000000000000..cc177c729f5af --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/cross_compatibility/network_direction.spec.ts @@ -0,0 +1,226 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { expect } from '@kbn/scout/api'; +import type { NetworkDirectionProcessor, StreamlangDSL } from '@kbn/streamlang'; +import { transpileEsql, transpileIngestPipeline } from '@kbn/streamlang'; +import { streamlangApiTest as apiTest } from '../..'; + +apiTest.describe( + 'Cross-compatibility - Network Direction Processor', + { tag: ['@ess', '@svlOblt'] }, + () => { + apiTest( + 'should set network direction with internal networks in both ingest pipeline and ES|QL', + async ({ testBed, esql }) => { + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks: ['private'], + } as NetworkDirectionProcessor, + ], + }; + + const { processors } = transpileIngestPipeline(streamlangDSL); + const { query } = transpileEsql(streamlangDSL); + + const docs = [ + { + source_ip: '128.232.110.120', + destination_ip: '192.168.1.1', + }, + ]; + + await testBed.ingest('ingest-e2e-test-network-direction-basic', docs, processors); + const ingestResult = await testBed.getDocs('ingest-e2e-test-network-direction-basic'); + + await testBed.ingest('esql-e2e-test-network-direction-basic', docs); + const esqlResult = await esql.queryOnIndex('esql-e2e-test-network-direction-basic', query); + + expect(ingestResult).toHaveLength(1); + expect(ingestResult[0]?.network.direction).toBe('inbound'); + expect(esqlResult.documents).toHaveLength(1); + expect(esqlResult.documents[0]?.['network.direction']).toBe('inbound'); + } + ); + + apiTest( + 'should set network direction with target field in both ingest pipeline and ES|QL', + async ({ testBed, esql }) => { + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + target_field: 'test_network_direction', + internal_networks: ['private'], + } as NetworkDirectionProcessor, + ], + }; + + const { processors } = transpileIngestPipeline(streamlangDSL); + const { query } = transpileEsql(streamlangDSL); + + const docs = [{ source_ip: '128.232.110.120', destination_ip: '192.168.1.1' }]; + await testBed.ingest('ingest-e2e-test-network-direction-target-field', docs, processors); + const ingestResult = await testBed.getDocs( + 'ingest-e2e-test-network-direction-target-field' + ); + + await testBed.ingest('esql-e2e-test-network-direction-target-field', docs); + const esqlResult = await esql.queryOnIndex( + 'esql-e2e-test-network-direction-target-field', + query + ); + + expect(ingestResult).toHaveLength(1); + expect(ingestResult[0]?.test_network_direction).toBe('inbound'); + expect(esqlResult.documents).toHaveLength(1); + expect(esqlResult.documents[0]?.test_network_direction).toBe('inbound'); + } + ); + + apiTest( + 'should set network direction with internal networks field in both ingest pipeline and ES|QL', + async ({ testBed, esql }) => { + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks_field: 'internal_networks', + } as NetworkDirectionProcessor, + ], + }; + + const { processors } = transpileIngestPipeline(streamlangDSL); + const { query } = transpileEsql(streamlangDSL); + + const docs = [ + { + source_ip: '128.232.110.120', + destination_ip: '192.168.1.1', + internal_networks: ['private'], + }, + ]; + + await testBed.ingest( + 'ingest-e2e-test-network-direction-internal-networks-field', + docs, + processors + ); + const ingestResult = await testBed.getDocs( + 'ingest-e2e-test-network-direction-internal-networks-field' + ); + + await testBed.ingest('esql-e2e-test-network-direction-internal-networks-field', docs); + const esqlResult = await esql.queryOnIndex( + 'esql-e2e-test-network-direction-internal-networks-field', + query + ); + + expect(ingestResult).toHaveLength(1); + expect(ingestResult[0]?.network.direction).toBe('inbound'); + expect(esqlResult.documents).toHaveLength(1); + expect(esqlResult.documents[0]?.['network.direction']).toBe('inbound'); + } + ); + + apiTest( + 'should set network direction with ignore_missing option in both ingest pipeline and ES|QL', + async ({ testBed, esql }) => { + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks: ['private'], + ignore_missing: true, + } as NetworkDirectionProcessor, + ], + }; + + const { processors } = transpileIngestPipeline(streamlangDSL); + const { query } = transpileEsql(streamlangDSL); + + const docs = [ + { source_ip: '128.232.110.120', destination_ip: '192.168.1.1' }, + { destination_ip: '192.168.1.1' }, + ]; + + await testBed.ingest('ingest-e2e-test-network-direction-ignore-missing', docs, processors); + const ingestResult = await testBed.getDocs( + 'ingest-e2e-test-network-direction-ignore-missing' + ); + + await testBed.ingest('esql-e2e-test-network-direction-ignore-missing', docs); + const esqlResult = await esql.queryOnIndex( + 'esql-e2e-test-network-direction-ignore-missing', + query + ); + + expect(ingestResult).toHaveLength(2); + expect(ingestResult[0]?.network.direction).toBe('inbound'); + expect(ingestResult[1]?.network?.direction).toBeUndefined(); + expect(esqlResult.documents).toHaveLength(2); + expect(esqlResult.documents[0]?.['network.direction']).toBe('inbound'); + expect(esqlResult.documents[1]?.['network.direction']).toBeNull(); + } + ); + + apiTest( + 'should set network direction with where condition in both ingest pipeline and ES|QL', + async ({ testBed, esql }) => { + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks: ['private'], + where: { + field: 'event.kind', + eq: 'test', + }, + } as NetworkDirectionProcessor, + ], + }; + + const { processors } = transpileIngestPipeline(streamlangDSL); + const { query } = transpileEsql(streamlangDSL); + + const docs = [ + { source_ip: '128.232.110.120', destination_ip: '192.168.1.1', event: { kind: 'test' } }, + { + source_ip: '128.232.110.120', + destination_ip: '192.168.1.1', + event: { kind: 'production' }, + }, + ]; + + await testBed.ingest('ingest-e2e-test-network-direction-where', docs, processors); + const ingestResult = await testBed.getDocs('ingest-e2e-test-network-direction-where'); + + await testBed.ingest('esql-e2e-test-network-direction-where', docs); + const esqlResult = await esql.queryOnIndex('esql-e2e-test-network-direction-where', query); + + expect(ingestResult).toHaveLength(2); + expect(ingestResult[0]?.network.direction).toBe('inbound'); + expect(ingestResult[1]?.network?.direction).toBeUndefined(); + expect(esqlResult.documents).toHaveLength(2); + expect(esqlResult.documents[0]?.['network.direction']).toBe('inbound'); + expect(esqlResult.documents[1]?.['network.direction']).toBeNull(); + } + ); + } +); diff --git a/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/esql/network_direction.spec.ts b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/esql/network_direction.spec.ts new file mode 100644 index 0000000000000..90be1f2627853 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/esql/network_direction.spec.ts @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { expect } from '@kbn/scout/api'; +import type { NetworkDirectionProcessor, StreamlangDSL } from '@kbn/streamlang'; +import { transpileEsql as transpile } from '@kbn/streamlang'; +import { streamlangApiTest as apiTest } from '../..'; + +apiTest.describe( + 'Streamlang to ES|QL - Network Direction Processor', + { tag: ['@ess', '@svlOblt'] }, + () => { + apiTest('should set network direction with internal networks', async ({ testBed, esql }) => { + const indexName = 'stream-e2e-test-network-direction-basic'; + + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks: ['private'], + } as NetworkDirectionProcessor, + ], + }; + + const { query } = transpile(streamlangDSL); + + const docs = [{ source_ip: '128.232.110.120', destination_ip: '192.168.1.1' }]; + await testBed.ingest(indexName, docs); + const esqlResult = await esql.queryOnIndex(indexName, query); + + expect(esqlResult.documents).toHaveLength(1); + expect(esqlResult.documents[0]?.['network.direction']).toBe('inbound'); + }); + + apiTest( + 'should set network direction with internal networks field', + async ({ testBed, esql }) => { + const indexName = 'stream-e2e-test-network-direction-internal-networks-field'; + + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks_field: 'test_network_direction_field', + } as NetworkDirectionProcessor, + ], + }; + + const { query } = transpile(streamlangDSL); + + const docs = [ + { + source_ip: '128.232.110.120', + destination_ip: '192.168.1.1', + test_network_direction_field: ['private'], + }, + ]; + await testBed.ingest(indexName, docs); + const esqlResult = await esql.queryOnIndex(indexName, query); + + expect(esqlResult.documents).toHaveLength(1); + expect(esqlResult.documents[0]?.['network.direction']).toBe('inbound'); + } + ); + + apiTest( + 'should set network direction with ignore_missing option', + async ({ testBed, esql }) => { + const indexName = 'stream-e2e-test-network-direction-ignore-missing'; + + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks: ['private'], + ignore_missing: true, + } as NetworkDirectionProcessor, + ], + }; + + const { query } = transpile(streamlangDSL); + + const docs = [ + { source_ip: '128.232.110.120', destination_ip: '192.168.1.1' }, + { destination_ip: '192.168.1.1' }, + ]; + await testBed.ingest(indexName, docs); + const esqlResult = await esql.queryOnIndex(indexName, query); + + expect(esqlResult.documents).toHaveLength(2); + expect(esqlResult.documents[0]?.['network.direction']).toBe('inbound'); + expect(esqlResult.documents[1]?.['network.direction']).toBeNull(); + } + ); + + apiTest('should set network direction with where condition', async ({ testBed, esql }) => { + const indexName = 'stream-e2e-test-network-direction-where'; + + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks: ['private'], + where: { + field: 'event.kind', + eq: 'test', + }, + } as NetworkDirectionProcessor, + ], + }; + + const { query } = transpile(streamlangDSL); + + const docs = [ + { source_ip: '128.232.110.120', destination_ip: '192.168.1.1', event: { kind: 'test' } }, + { + source_ip: '128.232.110.120', + destination_ip: '192.168.1.1', + event: { kind: 'production' }, + }, + ]; + await testBed.ingest(indexName, docs); + const esqlResult = await esql.queryOnIndex(indexName, query); + + expect(esqlResult.documents).toHaveLength(2); + expect(esqlResult.documents[0]?.['network.direction']).toBe('inbound'); + expect(esqlResult.documents[1]?.['network.direction']).toBeNull(); + }); + } +); diff --git a/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/ingest_pipeline/network_direction.spec.ts b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/ingest_pipeline/network_direction.spec.ts new file mode 100644 index 0000000000000..1f84aea94ab32 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/ingest_pipeline/network_direction.spec.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { expect } from '@kbn/scout/api'; +import type { NetworkDirectionProcessor, StreamlangDSL } from '@kbn/streamlang'; +import { transpile } from '@kbn/streamlang/src/transpilers/ingest_pipeline'; +import { streamlangApiTest as apiTest } from '../..'; + +apiTest.describe( + 'Streamlang to Ingest Pipeline - Network Direction Processor', + { tag: ['@ess', '@svlOblt'] }, + () => { + apiTest('should set network direction with internal networks', async ({ testBed }) => { + const indexName = 'streams-e2e-test-network-direction-basic'; + + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks: ['private'], + } as NetworkDirectionProcessor, + ], + }; + + const { processors } = transpile(streamlangDSL); + + const docs = [{ source_ip: '128.232.110.120', destination_ip: '192.168.1.1' }]; + await testBed.ingest(indexName, docs, processors); + + const ingestedDocs = await testBed.getDocs(indexName); + expect(ingestedDocs).toHaveLength(1); + expect(ingestedDocs[0]?.network.direction).toBe('inbound'); + }); + + apiTest('should set network direction with internal networks field', async ({ testBed }) => { + const indexName = 'streams-e2e-test-network-direction-internal-networks-field'; + + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks_field: 'internal_networks', + } as NetworkDirectionProcessor, + ], + }; + + const { processors } = transpile(streamlangDSL); + + const docs = [ + { + source_ip: '128.232.110.120', + destination_ip: '192.168.1.1', + internal_networks: ['private'], + }, + ]; + await testBed.ingest(indexName, docs, processors); + + const ingestedDocs = await testBed.getDocs(indexName); + expect(ingestedDocs).toHaveLength(1); + expect(ingestedDocs[0]?.network.direction).toBe('inbound'); + }); + + apiTest('should set network direction with target field', async ({ testBed }) => { + const indexName = 'streams-e2e-test-network-direction-target-field'; + + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + target_field: 'test_network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks: ['private'], + } as NetworkDirectionProcessor, + ], + }; + + const { processors } = transpile(streamlangDSL); + + const docs = [{ source_ip: '128.232.110.120', destination_ip: '192.168.1.1' }]; + await testBed.ingest(indexName, docs, processors); + + const ingestedDocs = await testBed.getDocs(indexName); + expect(ingestedDocs).toHaveLength(1); + expect(ingestedDocs[0]?.test_network_direction).toBe('inbound'); + }); + + apiTest( + 'should set network direction conditionally with where condition', + async ({ testBed }) => { + const indexName = 'streams-e2e-test-network-direction-conditional-where'; + + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks: ['private'], + where: { + field: 'event.kind', + eq: 'test', + }, + } as NetworkDirectionProcessor, + ], + }; + + const { processors } = transpile(streamlangDSL); + + const docs = [ + { source_ip: '128.232.110.120', destination_ip: '192.168.1.1', event: { kind: 'test' } }, + { + source_ip: '128.232.110.120', + destination_ip: '192.168.1.1', + event: { kind: 'production' }, + }, + ]; + await testBed.ingest(indexName, docs, processors); + + const ingestedDocs = await testBed.getDocs(indexName); + expect(ingestedDocs).toHaveLength(2); + expect(ingestedDocs[0]?.network.direction).toBe('inbound'); + expect(ingestedDocs[1]?.network?.direction).toBeUndefined(); + } + ); + + apiTest( + 'should omit network direction if any fields are missing with ignore_missing option', + async ({ testBed }) => { + const indexName = 'streams-e2e-test-network-direction-ignore-missing'; + + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks: ['private'], + ignore_missing: true, + } as NetworkDirectionProcessor, + ], + }; + + const { processors } = transpile(streamlangDSL); + + const docs = [ + { source_ip: '128.232.110.120', destination_ip: '192.168.1.1' }, + { source_ip: '128.232.110.120' }, + ]; + await testBed.ingest(indexName, docs, processors); + + const ingestedDocs = await testBed.getDocs(indexName); + expect(ingestedDocs).toHaveLength(2); + expect(ingestedDocs[0]?.network.direction).toBe('inbound'); + expect(ingestedDocs[1]?.network?.direction).toBeUndefined(); + } + ); + } +); diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/actions/action_metadata.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/actions/action_metadata.ts index e2bb0cca2e0d5..d392f254ca0e4 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/actions/action_metadata.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/actions/action_metadata.ts @@ -696,6 +696,36 @@ export const ACTION_METADATA_MAP: Record = { ], }, + network_direction: { + name: i18n.translate('xpack.streamlang.actionMetadata.networkDirection.name', { + defaultMessage: 'Network Direction', + }), + description: i18n.translate('xpack.streamlang.actionMetadata.networkDirection.description', { + defaultMessage: + 'Calculates the network direction given a source IP address, destination IP address, and a list of internal networks.', + }), + usage: i18n.translate('xpack.streamlang.actionMetadata.networkDirection.usage', { + defaultMessage: + 'Provide a `source_ip` and `destination_ip` field to specify the source and destination IP addresses. Use `internal_networks` or `internal_networks_field` to specify the list of internal networks.', + }), + examples: [ + { + description: i18n.translate( + 'xpack.streamlang.actionMetadata.networkDirection.examples.simple', + { + defaultMessage: + 'Calculate the network direction from a source IP address to a destination IP address', + } + ), + yaml: `action: network_direction + source_ip: attributes.source_ip + destination_ip: attributes.destination_ip + internal_networks: + - private`, + }, + ], + }, + manual_ingest_pipeline: { name: i18n.translate('xpack.streamlang.actionMetadata.manualIngestPipeline.name', { defaultMessage: 'Manual Ingest Pipeline', diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/__snapshots__/transpiler.test.ts.snap b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/__snapshots__/transpiler.test.ts.snap index 7b97a577bb8f2..a5319e9700df1 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/__snapshots__/transpiler.test.ts.snap +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/__snapshots__/transpiler.test.ts.snap @@ -21,7 +21,9 @@ Object { `; exports[`transpile - Streamlang DSL to ES|QL) should transpile a variety of processor steps and where blocks 1`] = ` -" | EVAL my_joined_field = CONCAT(field1, \\", \\", field2, \\", \\", field3) +" | WHERE NOT(source_ip IS NULL) AND NOT(destination_ip IS NULL) + | EVAL \`network.direction\` = NETWORK_DIRECTION(TO_IP(source_ip), TO_IP(destination_ip), [\\"private\\"]) + | EVAL my_joined_field = CONCAT(field1, \\", \\", field2, \\", \\", field3) | EVAL full_name = CASE(first_name IS NULL OR last_name IS NULL, NULL, CONCAT(first_name, \\" \\", last_name)) | WHERE NOT(message IS NULL) | EVAL message_uppercase = TO_UPPER(message) diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/conversions.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/conversions.ts index a7a8ba3a0e405..fd4d63ebb629e 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/conversions.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/conversions.ts @@ -29,6 +29,7 @@ import type { TrimProcessor, JoinProcessor, ConcatProcessor, + NetworkDirectionProcessor, } from '../../../types/processors'; import { type StreamlangProcessorDefinition } from '../../../types/processors'; import { convertRenameProcessorToESQL } from './processors/rename'; @@ -47,6 +48,7 @@ import { convertMathProcessorToESQL } from './processors/math'; import { createTransformStringESQL } from './transform_string'; import { convertJoinProcessorToESQL } from './processors/join'; import { convertConcatProcessorToESQL } from './processors/concat'; +import { convertNetworkDirectionProcessorToESQL } from './processors/network_direction'; function convertProcessorToESQL(processor: StreamlangProcessorDefinition): ESQLAstCommand[] | null { switch (processor.action) { @@ -107,6 +109,9 @@ function convertProcessorToESQL(processor: StreamlangProcessorDefinition): ESQLA case 'concat': return convertConcatProcessorToESQL(processor as ConcatProcessor); + case 'network_direction': + return convertNetworkDirectionProcessorToESQL(processor as NetworkDirectionProcessor); + case 'manual_ingest_pipeline': return [ Builder.command({ diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/common.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/common.ts index c87f53a4a3a6d..41cc1276a3aed 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/common.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/common.ts @@ -78,24 +78,43 @@ export function combineOr(predicates: ESQLAstItem[]): ESQLAstItem | null { * - ES|QL can only simulate the missing field case with WHERE filtering * - Pattern mismatch failures cannot be (in a reasonable fashion) simulated in ES|QL (they nullify target fields instead) * - * @param sourceField - The source field name to check for NULL/missing values * @param ignoreMissing - If false, returns WHERE command to filter missing fields + * @param sourceFields - The source field names to check for NULL/missing values * @returns WHERE command if filtering needed, undefined otherwise + * + * @example + * // Single field + * buildIgnoreMissingFilter(false, 'message') + * // → WHERE NOT(`message` IS NULL) + * + * @example + * // Multiple fields (e.g., network_direction) + * buildIgnoreMissingFilter(false, 'source.ip', 'destination.ip') + * // → WHERE NOT(`source.ip` IS NULL) AND NOT(`destination.ip` IS NULL) */ export function buildIgnoreMissingFilter( - sourceField: string, - ignoreMissing: boolean + ignoreMissing: boolean, + ...sourceFields: string[] ): ESQLAstCommand | undefined { - if (ignoreMissing) { + if (ignoreMissing || sourceFields.length === 0) { return undefined; // No filtering needed when ignore_missing = true } - const fromColumn = Builder.expression.column(sourceField); + const notNullPredicates = sourceFields.map((field) => + Builder.expression.func.call('NOT', [ + Builder.expression.func.postfix('IS NULL', Builder.expression.column(field)), + ]) + ); + + const combinedPredicate = combineAnd(notNullPredicates); + + if (!combinedPredicate) { + return undefined; + } + return Builder.command({ name: 'where', - args: [ - Builder.expression.func.call('NOT', [Builder.expression.func.postfix('IS NULL', fromColumn)]), - ], + args: [combinedPredicate], }); } diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/convert.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/convert.ts index 6e1d9fbb09d83..aac5fbf36b69d 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/convert.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/convert.ts @@ -58,7 +58,7 @@ export function convertConvertProcessorToESQL(processor: ConvertProcessor): ESQL const convertAssignment = Builder.expression.func.call(typeConversionFunction, [fromColumn]); // Add missing field filter if needed (ignore_missing = false) - const missingFieldFilter = buildIgnoreMissingFilter(from, ignore_missing); + const missingFieldFilter = buildIgnoreMissingFilter(ignore_missing, from); if (missingFieldFilter) { commands.push(missingFieldFilter); } diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/dissect.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/dissect.ts index cb9e6e9b0be75..e5c500a141f02 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/dissect.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/dissect.ts @@ -69,7 +69,7 @@ export function convertDissectProcessorToESQL(processor: DissectProcessor): ESQL const commands: ESQLAstCommand[] = []; // Add missing field filter if needed (ignore_missing = false) - const missingFieldFilter = buildIgnoreMissingFilter(from, ignore_missing); + const missingFieldFilter = buildIgnoreMissingFilter(ignore_missing, from); if (missingFieldFilter) { commands.push(missingFieldFilter); } diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/grok.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/grok.ts index 95494abc82133..afdd617dfdc72 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/grok.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/grok.ts @@ -67,7 +67,7 @@ export function convertGrokProcessorToESQL(processor: GrokProcessor): ESQLAstCom const commands: ESQLAstCommand[] = []; // Add missing field filter if needed (ignore_missing = false) - const missingFieldFilter = buildIgnoreMissingFilter(from, ignore_missing); + const missingFieldFilter = buildIgnoreMissingFilter(ignore_missing, from); if (missingFieldFilter) { commands.push(missingFieldFilter); } diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/network_direction.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/network_direction.ts new file mode 100644 index 0000000000000..4a909001e73bd --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/network_direction.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Builder, type ESQLAstCommand, type ESQLAstItem } from '@kbn/esql-language'; +import type { NetworkDirectionProcessor } from '../../../../types/processors'; +import { buildIgnoreMissingFilter } from './common'; +import { conditionToESQLAst } from '../condition_to_esql'; + +const DEFAULT_TARGET_FIELD = 'network.direction'; + +/** + * Converts a Streamlang NetworkDirectionProcessor into a list of ES|QL AST commands. + * + * @param processor - The NetworkDirectionProcessor to convert + * @returns A list of ES|QL AST commands + * @example + * Input: + * ``` + * { + * source_ip: '128.232.110.120', + * destination_ip: '192.168.1.1', + * internal_networks: ['private'], + * } + * ``` + * Output: + * ``` + * | EVAL network.direction = NETWORK_DIRECTION('128.232.110.120', '192.168.1.1', ['private']) + */ +export const convertNetworkDirectionProcessorToESQL = ( + processor: NetworkDirectionProcessor +): ESQLAstCommand[] => { + const { + source_ip, + destination_ip, + target_field = DEFAULT_TARGET_FIELD, + ignore_missing = false, + } = processor; + + const commands: ESQLAstCommand[] = []; + + const networkDirectionFuncArgs: ESQLAstItem[] = []; + // Wrap IP fields with TO_IP() to ensure type compatibility + // ES|QL NETWORK_DIRECTION requires ip type + networkDirectionFuncArgs.push( + Builder.expression.func.call('TO_IP', [Builder.expression.column(source_ip)]) + ); + networkDirectionFuncArgs.push( + Builder.expression.func.call('TO_IP', [Builder.expression.column(destination_ip)]) + ); + const networksArg = + 'internal_networks' in processor + ? Builder.expression.list.literal({ + values: processor.internal_networks.map((network) => + Builder.expression.literal.string(network) + ), + }) + : Builder.expression.column(processor.internal_networks_field); + networkDirectionFuncArgs.push(networksArg); + + const toColumn = Builder.expression.column(target_field); + + const networkDirectionFunc = Builder.expression.func.call('NETWORK_DIRECTION', [ + ...networkDirectionFuncArgs, + ]); + + const missingFieldFilter = buildIgnoreMissingFilter(ignore_missing, source_ip, destination_ip); + if (missingFieldFilter) { + commands.push(missingFieldFilter); + } + + if ('where' in processor && processor.where && !('always' in processor.where)) { + const conditionExpression = conditionToESQLAst(processor.where); + const caseExpression = Builder.expression.func.call('CASE', [ + conditionExpression, + networkDirectionFunc, + Builder.expression.literal.nil(), + ]); + commands.push( + Builder.command({ + name: 'eval', + args: [Builder.expression.func.binary('=', [toColumn, caseExpression])], + }) + ); + } else { + commands.push( + Builder.command({ + name: 'eval', + args: [Builder.expression.func.binary('=', [toColumn, networkDirectionFunc])], + }) + ); + } + + return commands; +}; diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/redact.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/redact.ts index 7118a12c5d720..9aef906392d45 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/redact.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/redact.ts @@ -106,7 +106,7 @@ export function convertRedactProcessorToESQL(processor: RedactProcessor): ESQLAs const commands: ESQLAstCommand[] = []; // Add missing field filter if needed (ignore_missing = false) - const missingFieldFilter = buildIgnoreMissingFilter(from, ignore_missing); + const missingFieldFilter = buildIgnoreMissingFilter(ignore_missing, from); if (missingFieldFilter) { commands.push(missingFieldFilter); } diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/remove.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/remove.ts index eeb177bc306ff..effe196fead96 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/remove.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/remove.ts @@ -91,7 +91,7 @@ export function convertRemoveProcessorToESQL(processor: RemoveProcessor): ESQLAs } else { // Unconditional removal: use DROP command // Add missing field filter if needed (ignore_missing = false) - const missingFieldFilter = buildIgnoreMissingFilter(from, ignore_missing); + const missingFieldFilter = buildIgnoreMissingFilter(ignore_missing, from); if (missingFieldFilter) { commands.push(missingFieldFilter); } diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/rename.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/rename.ts index b1067a37e60cb..c4aad8b6508e1 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/rename.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/rename.ts @@ -35,7 +35,7 @@ export function convertRenameProcessorToESQL(processor: RenameProcessor): ESQLAs const commands: ESQLAstCommand[] = []; // Add missing field filter if needed (ignore_missing = false) - const missingFieldFilter = buildIgnoreMissingFilter(from, ignore_missing); + const missingFieldFilter = buildIgnoreMissingFilter(ignore_missing, from); if (missingFieldFilter) { commands.push(missingFieldFilter); } diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/replace.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/replace.ts index 1abf4d97bc84e..4914cc8ec851f 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/replace.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/replace.ts @@ -101,7 +101,7 @@ export function convertReplaceProcessorToESQL(processor: ReplaceProcessor): ESQL const commands: ESQLAstCommand[] = []; // Add missing field filter if needed (ignore_missing = false) - const missingFieldFilter = buildIgnoreMissingFilter(from, ignore_missing); + const missingFieldFilter = buildIgnoreMissingFilter(ignore_missing, from); if (missingFieldFilter) { commands.push(missingFieldFilter); } diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/transform_string.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/transform_string.ts index 3385d80379bee..280c1cfbf2ed8 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/transform_string.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/transform_string.ts @@ -22,7 +22,7 @@ export const createTransformStringESQL = (esqlFunc: string) => { const commands: ESQLAstCommand[] = []; // Add missing field filter if needed (ignore_missing = false) - const missingFieldFilter = buildIgnoreMissingFilter(from, ignore_missing); + const missingFieldFilter = buildIgnoreMissingFilter(ignore_missing, from); if (missingFieldFilter) { commands.push(missingFieldFilter); } diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/ingest_pipeline/__snapshots__/transpiler.test.ts.snap b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/ingest_pipeline/__snapshots__/transpiler.test.ts.snap index 47fa36fc0dffa..163739d045961 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/ingest_pipeline/__snapshots__/transpiler.test.ts.snap +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/ingest_pipeline/__snapshots__/transpiler.test.ts.snap @@ -222,6 +222,15 @@ Object { exports[`transpile (Streamlang DSL to ingest pipeline) should transpile a variety of processor steps and where blocks 1`] = ` Object { "processors": Array [ + Object { + "network_direction": Object { + "destination_ip": "destination_ip", + "internal_networks": Array [ + "private", + ], + "source_ip": "source_ip", + }, + }, Object { "script": Object { "if": undefined, diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/ingest_pipeline/processors/pre_processing.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/ingest_pipeline/processors/pre_processing.ts index d4c7c06804eef..0a7796dae723d 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/ingest_pipeline/processors/pre_processing.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/ingest_pipeline/processors/pre_processing.ts @@ -31,6 +31,7 @@ export const processorFieldRenames: Record> = { trim: { from: 'field', to: 'target_field', where: 'if' }, join: { to: 'field', where: 'if' }, concat: { to: 'field', where: 'if' }, + network_direction: { where: 'if' }, manual_ingest_pipeline: { where: 'if' }, }; diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/ingest_pipeline/processors/processor.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/ingest_pipeline/processors/processor.ts index 95044b0cda50f..6df63e7accad5 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/ingest_pipeline/processors/processor.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/ingest_pipeline/processors/processor.ts @@ -25,6 +25,7 @@ import type { IngestPipelineTrimProcessor, IngestPipelineJoinProcessor, IngestPipelineConcatProcessor, + IngestPipelineNetworkDirectionProcessor, } from '../../../../types/processors/ingest_pipeline_processors'; type WithOptionalTracingTag = T & { tag?: string }; @@ -48,5 +49,6 @@ export interface ActionToIngestType { replace: WithOptionalTracingTag; redact: WithOptionalTracingTag; concat: WithOptionalTracingTag; + network_direction: WithOptionalTracingTag; manual_ingest_pipeline: WithOptionalTracingTag; } diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/shared/mocks/test_dsls.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/shared/mocks/test_dsls.ts index 0cfed7cc9400e..f686b058f65bf 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/shared/mocks/test_dsls.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/shared/mocks/test_dsls.ts @@ -22,11 +22,18 @@ import type { TrimProcessor, JoinProcessor, ConcatProcessor, + NetworkDirectionProcessor, } from '../../../../types/processors'; import type { StreamlangDSL } from '../../../../types/streamlang'; export const comprehensiveTestDSL: StreamlangDSL = { steps: [ + { + action: 'network_direction', + source_ip: 'source_ip', + destination_ip: 'destination_ip', + internal_networks: ['private'], + } as NetworkDirectionProcessor, { action: 'join', from: ['field1', 'field2', 'field3'], diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/validation/validate_field_names.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/validation/validate_field_names.ts index d47a6c603e2f0..4561ec7715390 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/validation/validate_field_names.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/validation/validate_field_names.ts @@ -90,6 +90,13 @@ export function extractAllFieldNames(processor: StreamlangProcessorDefinition): if (from.type === 'field') fields.push(from.value); }); break; + case 'network_direction': + fields.push(processor.source_ip, processor.destination_ip); + if (processor.target_field) fields.push(processor.target_field); + if ('internal_networks_field' in processor && processor.internal_networks_field) { + fields.push(processor.internal_networks_field); + } + break; case 'drop_document': case 'manual_ingest_pipeline': // No field names to validate diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/validation/validate_processor_values.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/validation/validate_processor_values.ts index 2ac4ec6cb8bf9..a86969d03e039 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/validation/validate_processor_values.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/validation/validate_processor_values.ts @@ -65,6 +65,7 @@ export function validateProcessorValues( case 'trim': case 'join': case 'concat': + case 'network_direction': case 'manual_ingest_pipeline': // No value validation implemented for these processors yet break; diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/validation/validate_types.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/validation/validate_types.ts index eeda07139ee33..ff8896e0ae2af 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/validation/validate_types.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/validation/validate_types.ts @@ -126,6 +126,12 @@ export function extractModifiedFields(processor: StreamlangProcessorDefinition): } break; + case 'network_direction': + if (processor.target_field) { + fields.push(processor.target_field); + } + break; + case 'remove': case 'remove_by_prefix': case 'drop_document': @@ -244,6 +250,9 @@ export function getProcessorOutputType( case 'join': return 'string'; + case 'network_direction': + return 'string'; + case 'remove': case 'remove_by_prefix': case 'drop_document': @@ -329,6 +338,7 @@ export function getExpectedInputType( case 'remove': case 'remove_by_prefix': case 'drop_document': + case 'network_direction': case 'manual_ingest_pipeline': return null; default: { @@ -390,6 +400,12 @@ export function trackFieldTypesAndValidate(flattenedSteps: StreamlangProcessorDe ...step.from.filter((from) => from.type === 'field').map((from) => from.value) ); break; + case 'network_direction': + fieldsUsed.push(step.source_ip, step.destination_ip); + if ('internal_networks_field' in step && step.internal_networks_field) { + fieldsUsed.push(step.internal_networks_field); + } + break; case 'append': case 'drop_document': case 'manual_ingest_pipeline': diff --git a/x-pack/platform/packages/shared/kbn-streamlang/types/processors/index.ts b/x-pack/platform/packages/shared/kbn-streamlang/types/processors/index.ts index 869e3e5dfdf6d..4173c5640a48e 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/types/processors/index.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/types/processors/index.ts @@ -557,6 +557,53 @@ export const concatProcessorSchema = processorBaseWithWhereSchema.extend({ ignore_missing: z.optional(z.boolean()), }) satisfies z.Schema; +/** + * Network direction processor + */ + +export interface NetworkDirectionWithInternalNetworks { + internal_networks: string[]; +} + +const networkDirectionWithInternalNetworksSchema = z.object({ + internal_networks: z.array(z.string()), +}) satisfies z.Schema; + +export interface NetworkDirectionWithInternalNetworksField { + internal_networks_field: string; +} + +const networkDirectionWithInternalNetworksFieldSchema = z.object({ + internal_networks_field: StreamlangSourceField, +}) satisfies z.Schema; + +export interface NetworkDirectionCommonFields extends ProcessorBaseWithWhere { + action: 'network_direction'; + source_ip: string; + destination_ip: string; + target_field?: string; + ignore_missing?: boolean; +} + +const networkDirectionCommonFieldsSchema = processorBaseWithWhereSchema.extend({ + action: z.literal('network_direction'), + source_ip: StreamlangSourceField, + destination_ip: StreamlangSourceField, + target_field: z.optional(StreamlangTargetField), + ignore_missing: z.optional(z.boolean()), +}) satisfies z.Schema; + +export type NetworkDirectionProcessor = NetworkDirectionCommonFields & + (NetworkDirectionWithInternalNetworks | NetworkDirectionWithInternalNetworksField); + +export const networkDirectionProcessorSchema = z.intersection( + networkDirectionCommonFieldsSchema, + z.union([ + networkDirectionWithInternalNetworksSchema, + networkDirectionWithInternalNetworksFieldSchema, + ]) +) satisfies z.Schema; + export type StreamlangProcessorDefinition = | DateProcessor | DissectProcessor @@ -576,6 +623,7 @@ export type StreamlangProcessorDefinition = | TrimProcessor | JoinProcessor | ConcatProcessor + | NetworkDirectionProcessor | ManualIngestPipelineProcessor; export const streamlangProcessorSchema = z.union([ @@ -597,6 +645,7 @@ export const streamlangProcessorSchema = z.union([ joinProcessorSchema, convertProcessorSchema, concatProcessorSchema, + networkDirectionProcessorSchema, manualIngestPipelineProcessorSchema, ]); @@ -654,6 +703,11 @@ export const processorTypes: ProcessorType[] = ( // Handle ZodEffects (from .refine()) by unwrapping to get the base schema let baseSchema = '_def' in schema && 'schema' in schema._def ? schema._def.schema : schema; + // Handle ZodIntersection (from z.intersection()) by getting the left side which contains the action + if ('_def' in baseSchema && 'left' in baseSchema._def) { + baseSchema = baseSchema._def.left; + } + // Handle ZodUnion (from z.union()) by getting the first option's action // All options in the union should have the same action value if ('_def' in baseSchema && 'options' in baseSchema._def) { diff --git a/x-pack/platform/packages/shared/kbn-streamlang/types/processors/ingest_pipeline_processors.ts b/x-pack/platform/packages/shared/kbn-streamlang/types/processors/ingest_pipeline_processors.ts index c941fd9767aa0..bfab2a3c3f837 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/types/processors/ingest_pipeline_processors.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/types/processors/ingest_pipeline_processors.ts @@ -26,6 +26,7 @@ import type { LowercaseProcessor, JoinProcessor, ConcatProcessor, + NetworkDirectionProcessor, } from '.'; import type { Condition } from '../conditions'; @@ -141,6 +142,12 @@ export type IngestPipelineConcatProcessor = RenameFieldsAndRemoveAction< { to: 'field'; where: 'if' } >; +// Network Direction +export type IngestPipelineNetworkDirectionProcessor = RenameFieldsAndRemoveAction< + NetworkDirectionProcessor, + { where: 'if' } +>; + // Manual Ingest Pipeline (escape hatch) export type IngestPipelineManualIngestPipelineProcessor = RenameFieldsAndRemoveAction< ManualIngestPipelineProcessor, @@ -166,4 +173,5 @@ export type IngestPipelineProcessor = | IngestPipelineTrimProcessor | IngestPipelineJoinProcessor | IngestPipelineConcatProcessor + | IngestPipelineNetworkDirectionProcessor | IngestPipelineManualIngestPipelineProcessor; diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/editor.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/editor.tsx index f1559583ad4b6..3c8286998f2de 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/editor.tsx +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/editor.tsx @@ -60,6 +60,7 @@ import { SetProcessorForm } from './set'; import { TransformStringProcessorForm } from './transform_string'; import { ConcatProcessorForm } from './concat'; import { JoinProcessorForm } from './join'; +import { NetworkDirectionProcessorForm } from './network_direction'; export const ActionBlockEditor = forwardRef((props, ref) => { const { processorMetrics, stepRef } = props; @@ -203,6 +204,7 @@ export const ActionBlockEditor = forwardRef((p )} {type === 'concat' && } {type === 'join' && } + {type === 'network_direction' && } {!SPECIALISED_TYPES.includes(type) && ( )} diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/network_direction/index.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/network_direction/index.tsx new file mode 100644 index 0000000000000..fcc68e3d9c0c0 --- /dev/null +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/network_direction/index.tsx @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiSpacer } from '@elastic/eui'; +import React from 'react'; +import { IgnoreFailureToggle, IgnoreMissingToggle } from '../ignore_toggles'; +import { + SourceIpField, + DestinationIpField, + NetworkDirectionTargetField, +} from './network_direction_inputs'; +import { ProcessorConditionEditor } from '../processor_condition_editor'; +import { FieldsAccordion } from '../optional_fields_accordion'; +import { InternalNetworksSelector } from './internal_networks_selector'; + +export const NetworkDirectionProcessorForm = () => { + return ( + <> + + + + + + + + + + + + + + ); +}; diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/network_direction/internal_networks_selector.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/network_direction/internal_networks_selector.tsx new file mode 100644 index 0000000000000..d49c64c51c4a2 --- /dev/null +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/network_direction/internal_networks_selector.tsx @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiButtonIcon, + EuiCheckableCard, + EuiFieldText, + EuiFlexGroup, + EuiFlexItem, + EuiFormFieldset, + EuiFormRow, + EuiSpacer, + EuiTitle, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React, { Fragment, useState, type ReactNode } from 'react'; +import { Controller, useFieldArray, useFormContext } from 'react-hook-form'; +import type { NetworkDirectionFormState } from '../../../../types'; +import { ProcessorFieldSelector } from '../processor_field_selector'; + +interface InternalNetworksFieldInputProps { + index: number; + onRemove: (index: number) => void; +} + +const InternalNetworksFieldInput = ({ index, onRemove }: InternalNetworksFieldInputProps) => { + const { control } = useFormContext(); + + return ( + + + ( + + + + )} + /> + + + onRemove(index)} + aria-label={i18n.translate( + 'xpack.streams.streamDetailView.managementTab.enrichment.processor.networkDirectionsSelectorInternalNetworksRemoveButton', + { defaultMessage: 'Remove' } + )} + /> + + + ); +}; + +const InternalNetworksContent = () => { + const { fields, append, remove } = useFieldArray< + Pick + >({ + name: 'internal_networks', + }); + + const handleAdd = () => append({ value: '' }); + + const handleRemove = (index: number) => remove(index); + + return ( + + + {fields.map((field, index) => ( + + + + ))} + + + + ); +}; + +const InternalNetworksFieldContent = () => ( + +); + +interface InternalNetworksOptions { + id: string; + label: string; + content: ReactNode; +} + +const internalNetworksOptions: InternalNetworksOptions[] = [ + { + id: 'internal_networks', + label: i18n.translate( + 'xpack.streams.streamDetailView.managementTab.enrichment.processor.networkDirectionsSelectorInternalNetworksLabel', + { defaultMessage: 'Define them manually.' } + ), + content: , + }, + { + id: 'internal_networks_field', + label: i18n.translate( + 'xpack.streams.streamDetailView.managementTab.enrichment.processor.networkDirectionSelectorInternalNetworksFieldLabel', + { defaultMessage: 'Read them from a field.' } + ), + content: , + }, +]; + +export const InternalNetworksSelector = () => { + const { unregister, watch } = useFormContext(); + const internalNetworks = watch('internal_networks'); + const initialSelectedOption = internalNetworks ? 'internal_networks' : 'internal_networks_field'; + const [selectedOption, setSelectedOption] = useState(initialSelectedOption); + + const handleOptionChange = (optionId: string) => { + setSelectedOption(optionId); + const unregisterOption = + optionId === 'internal_networks' ? 'internal_networks_field' : 'internal_networks'; + unregister(unregisterOption); + }; + + return ( + + + {i18n.translate( + 'xpack.streams.streamDetailView.managementTab.enrichment.processor.networkDirectionInternalNetworksLabel', + { defaultMessage: 'How do you want to define internal networks?' } + )} + + + ), + }} + > + {internalNetworksOptions.map(({ id, label, content }) => ( + + handleOptionChange(id)} + > + {selectedOption === id && content} + + + + ))} + + ); +}; diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/network_direction/network_direction_inputs.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/network_direction/network_direction_inputs.tsx new file mode 100644 index 0000000000000..1b9fd4bbdc2bc --- /dev/null +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/network_direction/network_direction_inputs.tsx @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { EuiFormRow, EuiSuperSelect } from '@elastic/eui'; +import { Controller, useFormContext } from 'react-hook-form'; +import { FieldNameWithIcon } from '@kbn/react-field'; +import { useEnrichmentFieldSuggestions } from '../../../../../../../hooks/use_field_suggestions'; +import { ProcessorFieldSelector } from '../processor_field_selector'; + +interface IpRequiredFieldProps { + name: string; + label: string; + dataTestSubj: string; +} + +const IpRequiredField = ({ name, label, dataTestSubj }: IpRequiredFieldProps) => { + const { control } = useFormContext(); + const fieldSuggestions = useEnrichmentFieldSuggestions(); + const options = fieldSuggestions.map((suggestion) => ({ + value: suggestion.name, + inputDisplay: , + })); + + return ( + ( + + + + )} + /> + ); +}; + +export const SourceIpField = () => { + return ( + + ); +}; + +export const DestinationIpField = () => { + return ( + + ); +}; + +export const NetworkDirectionTargetField = () => { + return ( + + ); +}; diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/processor_type_selector.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/processor_type_selector.tsx index 0551a2f64948d..2e03303527c0a 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/processor_type_selector.tsx +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/processor_type_selector.tsx @@ -519,6 +519,37 @@ const getAvailableProcessors: ( ); }, }, + network_direction: { + type: 'network_direction' as const, + inputDisplay: i18n.translate( + 'xpack.streams.streamDetailView.managementTab.enrichment.processor.networkDirectionInputDisplay', + { + defaultMessage: 'Network direction', + } + ), + getDocUrl: (docLinks: DocLinksStart) => { + return ( + + {i18n.translate('xpack.streams.availableProcessors.networkDirectionLinkLabel', { + defaultMessage: 'Network direction', + })} + + ), + }} + /> + ); + }, + }, ...configDrivenProcessors, ...(isWired ? {} @@ -565,6 +596,7 @@ const PROCESSOR_GROUP_MAP: Record< trim: 'set', join: 'set', concat: 'set', + network_direction: 'set', }; const getProcessorDescription = diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/utils.ts b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/utils.ts index e655287206b44..9b6efc5e09a19 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/utils.ts +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/utils.ts @@ -188,6 +188,18 @@ export const getStepDescription = (step: StreamlangProcessorDefinitionWithUIAttr }, } ); + } else if (step.action === 'network_direction') { + const { source_ip, destination_ip } = step; + return i18n.translate( + 'xpack.streams.streamDetailView.managementTab.enrichment.networkDirectionProcessorDescription', + { + defaultMessage: 'Network direction from "{source_ip}" to "{destination_ip}".', + values: { + source_ip, + destination_ip, + }, + } + ); } else { const { action, parentId, customIdentifier, ignore_failure, ...rest } = step; // Remove 'where' if it exists (some processors have it, some don't) diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/types.ts b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/types.ts index dd18536581d20..d8375d85e6686 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/types.ts +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/types.ts @@ -22,6 +22,7 @@ import type { TrimProcessor, JoinProcessor, ConcatProcessor, + NetworkDirectionProcessor, } from '@kbn/streamlang'; import type { EnrichmentDataSource } from '../../../../common/url_schema'; import type { ConfigDrivenProcessorFormState } from './steps/blocks/action/config_driven/types'; @@ -39,6 +40,10 @@ export type GrokFormState = Omit & { patterns: GrokPatternField[]; }; +export interface InternalNetworksValue { + value: string; +} + export type DissectFormState = DissectProcessor; export type DateFormState = DateProcessor; export type DropFormState = DropDocumentProcessor; @@ -63,6 +68,13 @@ export type LowercaseFormState = LowercaseProcessor; export type TrimFormState = TrimProcessor; export type JoinFormState = JoinProcessor; export type ConcatFormState = ConcatProcessor; +export type NetworkDirectionFormState = Omit< + NetworkDirectionProcessor, + 'internal_networks' | 'internal_networks_field' +> & { + internal_networks?: InternalNetworksValue[]; + internal_networks_field?: string; +}; export type SpecialisedFormState = | GrokFormState @@ -79,7 +91,8 @@ export type SpecialisedFormState = | LowercaseFormState | TrimFormState | JoinFormState - | ConcatFormState; + | ConcatFormState + | NetworkDirectionFormState; export type ProcessorFormState = SpecialisedFormState | ConfigDrivenProcessorFormState; export type ConditionBlockFormState = StreamlangConditionBlockWithUIAttributes; diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/utils.ts b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/utils.ts index d8d3451ddb940..53bc78c9df7a6 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/utils.ts +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/utils.ts @@ -13,6 +13,7 @@ import type { JoinProcessor, LowercaseProcessor, MathProcessor, + NetworkDirectionProcessor, ProcessorType, RedactProcessor, ReplaceProcessor, @@ -34,7 +35,7 @@ import { isConditionBlock } from '@kbn/streamlang/types/streamlang'; import type { FlattenRecord } from '@kbn/streams-schema'; import { Streams, isSchema, type FieldDefinition } from '@kbn/streams-schema'; import type { IngestUpsertRequest } from '@kbn/streams-schema/src/models/ingest'; -import { countBy, isEmpty, mapValues, orderBy } from 'lodash'; +import { countBy, isEmpty, mapValues, omit, orderBy } from 'lodash'; import type { EnrichmentDataSource } from '../../../../common/url_schema'; import type { StreamEnrichmentContextType } from './state_management/stream_enrichment_state_machine/types'; import { configDrivenProcessors } from './steps/blocks/action/config_driven'; @@ -55,6 +56,7 @@ import type { LowercaseFormState, ManualIngestPipelineFormState, MathFormState, + NetworkDirectionFormState, ProcessorFormState, RedactFormState, ReplaceFormState, @@ -81,6 +83,7 @@ export const SPECIALISED_TYPES = [ 'trim', 'join', 'concat', + 'network_direction', ]; interface FormStateDependencies { @@ -291,6 +294,17 @@ const defaultConcatProcessorFormState = (): ConcatFormState => ({ where: ALWAYS_CONDITION, }); +const defaultNetworkDirectionProcessorFormState = (): NetworkDirectionFormState => ({ + action: 'network_direction' as const, + source_ip: '', + destination_ip: '', + internal_networks: [], + target_field: 'attributes.network.direction', + ignore_failure: true, + ignore_missing: true, + where: ALWAYS_CONDITION, +}); + const configDrivenDefaultFormStates = mapValues( configDrivenProcessors, (config) => () => config.defaultFormState @@ -317,6 +331,7 @@ const defaultProcessorFormStateByType: Record< set: defaultSetProcessorFormState, join: defaultJoinProcessorFormState, concat: defaultConcatProcessorFormState, + network_direction: defaultNetworkDirectionProcessorFormState, ...configDrivenDefaultFormStates, }; @@ -351,6 +366,24 @@ export const getFormStateFromActionStep = ( }; } + if (step.action === 'network_direction') { + const clone: NetworkDirectionFormState = structuredClone({ + ...omit(step, 'internal_networks', 'internal_networks_field'), + }); + + if ('internal_networks' in step) { + clone.internal_networks = step.internal_networks?.map((internalNetwork) => ({ + value: internalNetwork, + })); + } + + if ('internal_networks_field' in step) { + clone.internal_networks_field = step.internal_networks_field; + } + + return clone; + } + if ( step.action === 'dissect' || step.action === 'manual_ingest_pipeline' || @@ -677,6 +710,29 @@ export const convertFormStateToProcessor = ( }; } + if (formState.action === 'network_direction') { + const { source_ip, destination_ip, target_field, ignore_failure, ignore_missing } = formState; + + return { + processorDefinition: { + action: 'network_direction', + source_ip, + destination_ip, + internal_networks: + 'internal_networks' in formState + ? formState.internal_networks?.map((internalNetwork) => internalNetwork.value) + : undefined, + internal_networks_field: + 'internal_networks_field' in formState ? formState.internal_networks_field : undefined, + target_field, + ignore_failure, + ignore_missing, + description, + where: 'where' in formState ? formState.where : undefined, + } as NetworkDirectionProcessor, + }; + } + if (configDrivenProcessors[formState.action]) { return { processorDefinition: {