address comments

gpop63 · gpop63 · commit 53ee3b94a69c · 2025-11-18T12:14:11.000+02:00
diff --git a/packages/aws_bedrock_agentcore/_dev/build/docs/README.md b/packages/aws_bedrock_agentcore/_dev/build/docs/README.md
@@ -35,6 +35,17 @@ For more details about these requirements, check the [AWS integration documentat
 * You can install only one Elastic Agent per host.
 * Elastic Agent is required to collect metrics from CloudWatch and ship the data to Elastic, where the events will then be processed through the integration's ingest pipelines.
 
+### How to find the `log_group_arn` (for log-based datasets)
+
+Some datasets in this integration require the ARN of the CloudWatch log group where your AgentCore logs are stored. You can find it by:
+
+- Opening CloudWatch in the AWS Console
+- Going to Logs > Log groups
+- Selecting the log group used by your AgentCore deployment
+- Copying the Log group ARN shown
+
+You can then use this ARN when configuring any log-based dataset.
+
 ## Setup
 
 To use the Amazon Bedrock AgentCore metrics, ensure your agents are deployed and running. The integration will automatically collect metrics from the AWS/Bedrock-AgentCore CloudWatch namespace. For enhanced observability, enable detailed monitoring and logging for your AgentCore resources.
diff --git a/packages/aws_bedrock_agentcore/data_stream/runtime_application_logs/_dev/test/pipeline/test-aws-bedrock-agentcore.log-expected.json b/packages/aws_bedrock_agentcore/data_stream/runtime_application_logs/_dev/test/pipeline/test-aws-bedrock-agentcore.log-expected.json
@@ -26,61 +26,13 @@
                 "outcome": "success"
             },
             "gen_ai": {
-                "conversation_id": "090ab333-40ff-49cd-ade1-1527b3a8ede7",
-                "operation": {
-                    "name": "InvokeAgentRuntime"
-                },
-                "prompt": "What is this agent about, this is claudserver?",
-                "provider": {
-                    "name": "aws"
+                "conversation": {
+                    "id": "090ab333-40ff-49cd-ade1-1527b3a8ede7"
                 },
-                "system": "aws_bedrock_agentcore"
-            },
-            "service": {
-                "name": "AgentCoreCodeRuntime"
-            },
-            "span": {
-                "id": "3f2b2909093d27f3"
-            },
-            "tags": [
-                "preserve_original_event",
-                "preserve_duplicate_custom_fields"
-            ],
-            "trace": {
-                "id": "69082fe91a0ed1872de8e57378fe229b"
-            }
-        },
-        {
-            "@timestamp": "2025-11-03T04:30:39.907Z",
-            "aws": {
-                "bedrock_agentcore": {
-                    "account_id": "627286350134",
-                    "request_id": "136b5865-b303-43a7-b737-3fed1ba17341",
-                    "resource_arn": "arn:aws:bedrock-agentcore:us-east-1:627286350134:runtime/claudeserver-CdBoW2FLP0"
-                }
-            },
-            "cloud": {
-                "account": {
-                    "id": "627286350134"
-                },
-                "provider": "aws",
-                "service": {
-                    "name": "bedrock-agentcore"
-                }
-            },
-            "ecs": {
-                "version": "8.11.0"
-            },
-            "event": {
-                "original": "sha1-bbdf20a287f648bad430e94671e9da41e9c430c0:33460",
-                "outcome": "success"
-            },
-            "gen_ai": {
-                "conversation_id": "090ab333-40ff-49cd-ade1-1527b3a8ede7",
                 "operation": {
                     "name": "InvokeAgentRuntime"
                 },
-                "prompt_hash": "fa80527c3c86dbbca4fe8e6f605389c2cd02c1cc",
+                "prompt": "What is this agent about, this is claudserver?",
                 "provider": {
                     "name": "aws"
                 },
diff --git a/packages/aws_bedrock_agentcore/data_stream/runtime_application_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws_bedrock_agentcore/data_stream/runtime_application_logs/elasticsearch/ingest_pipeline/default.yml
@@ -87,7 +87,7 @@ processors:
   # Handle session information
   - rename:
       field: aws.bedrock_agentcore.session_id
-      target_field: gen_ai.conversation_id
+      target_field: gen_ai.conversation.id
       ignore_missing: true
   - rename:
       field: aws.bedrock_agentcore.region
@@ -112,7 +112,7 @@ processors:
       if: ctx.gen_ai?.prompt != null && ctx.gen_ai?.prompt instanceof String
       source: |
         if (ctx.gen_ai.prompt.length() > 32766) {
-          ctx.gen_ai.prompt_hash = ctx.gen_ai.prompt.sha1(); 
+          ctx.aws.bedrock_agentcore.prompt_hash = ctx.gen_ai.prompt.sha1(); 
           ctx.gen_ai.remove("prompt");
         }
       ignore_failure: true
@@ -155,12 +155,9 @@ processors:
       ignore_failure: true
 
 on_failure:
-  - set:
+   - set:
       field: error.message
-      value: >-
-        Processor '{{{ _ingest.on_failure_processor_type }}}'
-        {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
-        {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
+      value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
       field: event.outcome
       value: failure
diff --git a/packages/aws_bedrock_agentcore/data_stream/runtime_application_logs/fields/fields.yml b/packages/aws_bedrock_agentcore/data_stream/runtime_application_logs/fields/fields.yml
@@ -13,6 +13,10 @@
       type: keyword
       description: ID of the request.
 
+    - name: prompt_hash
+      type: text
+      description: Hash of the user prompt for deduplication purposes.
+
 # Gen AI fields relevant for AgentCore
 - name: gen_ai
   type: group
@@ -21,7 +25,7 @@
       type: keyword
       description: Name of the AI system (aws).
 
-    - name: conversation_id
+    - name: conversation.id
       type: keyword
       description: ID of the conversation.
 
@@ -33,18 +37,10 @@
       type: keyword
       description: Name of the provider.
 
-    - name: response.timestamp
-      type: date
-      description: Timestamp when the response was received.
-
     - name: prompt
       type: text
       description: The user prompt sent to the AgentCore runtime.
 
-    - name: prompt_hash
-      type: text
-      description: Hash of the user prompt for deduplication purposes.
-
     - name: request
       type: group
       fields:
diff --git a/packages/aws_bedrock_agentcore/docs/README.md b/packages/aws_bedrock_agentcore/docs/README.md
@@ -35,6 +35,17 @@ For more details about these requirements, check the [AWS integration documentat
 * You can install only one Elastic Agent per host.
 * Elastic Agent is required to collect metrics from CloudWatch and ship the data to Elastic, where the events will then be processed through the integration's ingest pipelines.
 
+### How to find the `log_group_arn` (for log-based datasets)
+
+Some datasets in this integration require the ARN of the CloudWatch log group where your AgentCore logs are stored. You can find it by:
+
+- Opening CloudWatch in the AWS Console
+- Going to Logs > Log groups
+- Selecting the log group used by your AgentCore deployment
+- Copying the Log group ARN shown
+
+You can then use this ARN when configuring any log-based dataset.
+
 ## Setup
 
 To use the Amazon Bedrock AgentCore metrics, ensure your agents are deployed and running. The integration will automatically collect metrics from the AWS/Bedrock-AgentCore CloudWatch namespace. For enhanced observability, enable detailed monitoring and logging for your AgentCore resources.
@@ -296,6 +307,7 @@ An example event for `runtime_application` looks as following:
 |---|---|---|
 | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date |
 | aws.bedrock_agentcore.account_id | ID of the account. | keyword |
+| aws.bedrock_agentcore.prompt_hash | Hash of the user prompt for deduplication purposes. | text |
 | aws.bedrock_agentcore.request_id | ID of the request. | keyword |
 | aws.bedrock_agentcore.resource_arn | The Amazon Resource Name (ARN) of the AgentCore runtime resource. | keyword |
 | aws.cloudwatch.message | CloudWatch log message. | text |
@@ -305,13 +317,11 @@ An example event for `runtime_application` looks as following:
 | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword |
 | event.dataset | Event dataset | constant_keyword |
 | event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword |
-| gen_ai.conversation_id | ID of the conversation. | keyword |
+| gen_ai.conversation.id | ID of the conversation. | keyword |
 | gen_ai.operation.name | Name of the operation. | keyword |
 | gen_ai.prompt | The user prompt sent to the AgentCore runtime. | text |
-| gen_ai.prompt_hash | Hash of the user prompt for deduplication purposes. | text |
 | gen_ai.provider.name | Name of the provider. | keyword |
 | gen_ai.request.id | Unique identifier for the request. | keyword |
-| gen_ai.response.timestamp | Timestamp when the response was received. | date |
 | gen_ai.system | Name of the AI system (aws). | keyword |
 | host.containerized | If the host is a container. | boolean |
 | host.os.build | OS build information. | keyword |
diff --git a/packages/aws_bedrock_agentcore/kibana/dashboard/aws_bedrock_agentcore-3235ab43-8fa0-47f5-9590-b8d7451a4c92.json b/packages/aws_bedrock_agentcore/kibana/dashboard/aws_bedrock_agentcore-3235ab43-8fa0-47f5-9590-b8d7451a4c92.json
@@ -832,7 +832,7 @@
                     "syncTooltips": false,
                     "filters": [],
                     "query": {
-                        "esql": "FROM logs-aws_bedrock_agentcore.runtime_application_logs-default |\nKEEP @timestamp, aws.bedrock_agentcore.request_id, aws.bedrock_agentcore.resource_arn, gen_ai.prompt, trace.id, gen_ai.conversation_id\n| SORT `@timestamp` desc\n| RENAME aws.bedrock_agentcore.request_id as `Request ID`, gen_ai.conversation_id as `Session ID`, trace.id as `Trace ID`, aws.bedrock_agentcore.resource_arn as `Resource ARN`, gen_ai.prompt as `Prompt`, @timestamp as `Timestamp`"
+                        "esql": "FROM logs-aws_bedrock_agentcore.runtime_application_logs-default |\nKEEP @timestamp, aws.bedrock_agentcore.request_id, aws.bedrock_agentcore.resource_arn, gen_ai.prompt, trace.id, gen_ai.conversation.id\n| SORT `@timestamp` desc\n| RENAME aws.bedrock_agentcore.request_id as `Request ID`, gen_ai.conversation.id as `Session ID`, trace.id as `Trace ID`, aws.bedrock_agentcore.resource_arn as `Resource ARN`, gen_ai.prompt as `Prompt`, @timestamp as `Timestamp`"
                     },
                     "attributes": {
                         "title": "Table Timestamp & Request ID & Resource ARN & Prompt & Trace ID",
@@ -844,7 +844,7 @@
                                         "0d3b9a8b-965d-40e4-b6cb-7559a35d0e73": {
                                             "index": "840dc00d90e0cb0f19f52fcd03c115341d02482231833d5487b35b9a5ed8b7de",
                                             "query": {
-                                                "esql": "FROM logs-aws_bedrock_agentcore.runtime_application_logs-default |\nKEEP @timestamp, aws.bedrock_agentcore.request_id, aws.bedrock_agentcore.resource_arn, gen_ai.prompt, trace.id, gen_ai.conversation_id\n| SORT `@timestamp` desc\n| RENAME aws.bedrock_agentcore.request_id as `Request ID`, gen_ai.conversation_id as `Session ID`, trace.id as `Trace ID`, aws.bedrock_agentcore.resource_arn as `Resource ARN`, gen_ai.prompt as `Prompt`, @timestamp as `Timestamp`"
+                                                "esql": "FROM logs-aws_bedrock_agentcore.runtime_application_logs-default |\nKEEP @timestamp, aws.bedrock_agentcore.request_id, aws.bedrock_agentcore.resource_arn, gen_ai.prompt, trace.id, gen_ai.conversation.id\n| SORT `@timestamp` desc\n| RENAME aws.bedrock_agentcore.request_id as `Request ID`, gen_ai.conversation.id as `Session ID`, trace.id as `Trace ID`, aws.bedrock_agentcore.resource_arn as `Resource ARN`, gen_ai.prompt as `Prompt`, @timestamp as `Timestamp`"
                                             },
                                             "columns": [
                                                 {
@@ -987,7 +987,7 @@
                             },
                             "filters": [],
                             "query": {
-                                "esql": "FROM logs-aws_bedrock_agentcore.runtime_application_logs-default |\nKEEP @timestamp, aws.bedrock_agentcore.request_id, aws.bedrock_agentcore.resource_arn, gen_ai.prompt, trace.id, gen_ai.conversation_id\n| SORT `@timestamp` desc\n| RENAME aws.bedrock_agentcore.request_id as `Request ID`, gen_ai.conversation_id as `Session ID`, trace.id as `Trace ID`, aws.bedrock_agentcore.resource_arn as `Resource ARN`, gen_ai.prompt as `Prompt`, @timestamp as `Timestamp`"
+                                "esql": "FROM logs-aws_bedrock_agentcore.runtime_application_logs-default |\nKEEP @timestamp, aws.bedrock_agentcore.request_id, aws.bedrock_agentcore.resource_arn, gen_ai.prompt, trace.id, gen_ai.conversation.id\n| SORT `@timestamp` desc\n| RENAME aws.bedrock_agentcore.request_id as `Request ID`, gen_ai.conversation.id as `Session ID`, trace.id as `Trace ID`, aws.bedrock_agentcore.resource_arn as `Resource ARN`, gen_ai.prompt as `Prompt`, @timestamp as `Timestamp`"
                             },
                             "visualization": {
                                 "layerId": "0d3b9a8b-965d-40e4-b6cb-7559a35d0e73",
