From abc75a791db5b66c5029c1a6a6103a2447a81918 Mon Sep 17 00:00:00 2001 From: Mohse Morad Date: Thu, 31 Jul 2025 21:49:45 +0300 Subject: [PATCH 01/11] Add test --- .../conversation_history.json | 257 ++++++++++++++++++ .../91_calling_datadog/test_case.yaml | 15 + tests/llm/utils/test_case_utils.py | 2 +- 3 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 tests/llm/fixtures/test_ask_holmes/91_calling_datadog/conversation_history.json create mode 100644 tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/conversation_history.json b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/conversation_history.json new file mode 100644 index 0000000000..281e47df00 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/conversation_history.json @@ -0,0 +1,257 @@ +[ + { + "role": "user", + "content": "what is unhealthy?", + "token_count": 11 + }, + { + "role": "assistant", + "tool_calls": + [ + { + "id": "call_3jdRhOTW5h3yaNlkhpbyvLN6", + "type": "function", + "function": + { + "name": "kubectl_get_by_kind_in_cluster", + "arguments": "{\"kind\":\"pod\"}" + } + } + ] + }, + { + "name": "kubectl_get_by_kind_in_cluster", + "role": "tool", + "content": "Params used for the tool call: {\"kind\": \"pod\"}. The tool call output follows on the next line.\nNAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES LABELS\ndefault alertmanager-robusta-kube-prometheus-st-alertmanager-0 2/2 Running 0 155d 172.31.40.168 ip-172-31-47-239.us-east-2.compute.internal alertmanager=robusta-kube-prometheus-st-alertmanager,app.kubernetes.io/instance=robusta-kube-prometheus-st-alertmanager,app.kubernetes.io/managed-by=prometheus-operator,app.kubernetes.io/name=alertmanager,app.kubernetes.io/version=0.26.0,apps.kubernetes.io/pod-index=0,controller-revision-hash=alertmanager-robusta-kube-prometheus-st-alertmanager-57cd7fb46f,statefulset.kubernetes.io/pod-name=alertmanager-robusta-kube-prometheus-st-alertmanager-0\ndefault curl-deployment-6c67b4656-nhjwj 1/1 Running 0 28h 172.31.12.12 ip-172-31-5-104.us-east-2.compute.internal app=curl-app,pod-template-hash=6c67b4656\ndefault datadog-agent-sqj8m 2/2 Running 0 45m 172.31.41.35 ip-172-31-47-239.us-east-2.compute.internal agent.datadoghq.com/component=agent,agent.datadoghq.com/name=datadog,agent.datadoghq.com/provider=,app.kubernetes.io/component=agent,app.kubernetes.io/instance=datadog-agent,app.kubernetes.io/managed-by=datadog-operator,app.kubernetes.io/name=datadog-agent-deployment,app.kubernetes.io/part-of=default-datadog,app.kubernetes.io/version=,controller-revision-hash=b5bff5c95,pod-template-generation=2\ndefault datadog-agent-tjhbs 2/2 Running 0 46m 172.31.20.130 ip-172-31-24-130.us-east-2.compute.internal agent.datadoghq.com/component=agent,agent.datadoghq.com/name=datadog,agent.datadoghq.com/provider=,app.kubernetes.io/component=agent,app.kubernetes.io/instance=datadog-agent,app.kubernetes.io/managed-by=datadog-operator,app.kubernetes.io/name=datadog-agent-deployment,app.kubernetes.io/part-of=default-datadog,app.kubernetes.io/version=,controller-revision-hash=b5bff5c95,pod-template-generation=2\ndefault datadog-agent-zzxns 2/2 Running 0 46m 172.31.0.245 ip-172-31-5-104.us-east-2.compute.internal agent.datadoghq.com/component=agent,agent.datadoghq.com/name=datadog,agent.datadoghq.com/provider=,app.kubernetes.io/component=agent,app.kubernetes.io/instance=datadog-agent,app.kubernetes.io/managed-by=datadog-operator,app.kubernetes.io/name=datadog-agent-deployment,app.kubernetes.io/part-of=default-datadog,app.kubernetes.io/version=,controller-revision-hash=b5bff5c95,pod-template-generation=2\ndefault datadog-cluster-agent-55d484796b-x6vl2 1/1 Running 0 68m 172.31.1.219 ip-172-31-5-104.us-east-2.compute.internal agent.datadoghq.com/component=cluster-agent,agent.datadoghq.com/name=datadog,app.kubernetes.io/component=cluster-agent,app.kubernetes.io/instance=datadog-cluster-agent,app.kubernetes.io/managed-by=datadog-operator,app.kubernetes.io/name=datadog-agent-deployment,app.kubernetes.io/part-of=default-datadog,app.kubernetes.io/version=,pod-template-hash=55d484796b\ndefault datadog-operator-6c9b79787f-bxhrk 1/1 Running 0 71m 172.31.10.101 ip-172-31-5-104.us-east-2.compute.internal app.kubernetes.io/instance=datadog-operator,app.kubernetes.io/name=datadog-operator,pod-template-hash=6c9b79787f\ndefault meme-deployment-74db7bc95c-dkdhk 1/1 Running 0 28h 172.31.12.146 ip-172-31-5-104.us-east-2.compute.internal app=meme-app,pod-template-hash=74db7bc95c\ndefault meme-deployment-74db7bc95c-vznd5 1/1 Running 0 28h 172.31.43.211 ip-172-31-47-239.us-east-2.compute.internal app=meme-app,pod-template-hash=74db7bc95c\ndefault prometheus-robusta-kube-prometheus-st-prometheus-0 2/2 Running 0 123d 172.31.3.100 ip-172-31-5-104.us-east-2.compute.internal app.kubernetes.io/instance=robusta-kube-prometheus-st-prometheus,app.kubernetes.io/managed-by=prometheus-operator,app.kubernetes.io/name=prometheus,app.kubernetes.io/version=2.48.1,apps.kubernetes.io/pod-index=0,controller-revision-hash=prometheus-robusta-kube-prometheus-st-prometheus-55d87c869b,operator.prometheus.io/name=robusta-kube-prometheus-st-prometheus,operator.prometheus.io/shard=0,prometheus=robusta-kube-prometheus-st-prometheus,statefulset.kubernetes.io/pod-name=prometheus-robusta-kube-prometheus-st-prometheus-0\ndefault robusta-forwarder-6759cbdf94-5q4qh 1/1 Running 0 36d 172.31.27.42 ip-172-31-24-130.us-east-2.compute.internal app=robusta-forwarder,pod-template-hash=6759cbdf94\ndefault robusta-grafana-8487fb665f-2l8ss 3/3 Running 0 149d 172.31.33.141 ip-172-31-47-239.us-east-2.compute.internal app.kubernetes.io/instance=robusta,app.kubernetes.io/name=grafana,pod-template-hash=8487fb665f\ndefault robusta-holmes-7799dc97f6-f5n86 1/1 Running 0 8m44s 172.31.23.25 ip-172-31-24-130.us-east-2.compute.internal app=holmes,pod-template-hash=7799dc97f6\ndefault robusta-kube-prometheus-st-operator-6885c8f675-25qfq 1/1 Running 0 179d 172.31.47.106 ip-172-31-47-239.us-east-2.compute.internal app.kubernetes.io/instance=robusta,app.kubernetes.io/managed-by=Helm,app.kubernetes.io/part-of=kube-prometheus-stack,app.kubernetes.io/version=55.7.0,app=kube-prometheus-stack-operator,chart=kube-prometheus-stack-55.7.0,heritage=Helm,pod-template-hash=6885c8f675,release=robusta\ndefault robusta-kube-state-metrics-8667fd9775-slv9m 1/1 Running 0 179d 172.31.8.207 ip-172-31-5-104.us-east-2.compute.internal app.kubernetes.io/component=metrics,app.kubernetes.io/instance=robusta,app.kubernetes.io/managed-by=Helm,app.kubernetes.io/name=kube-state-metrics,app.kubernetes.io/part-of=kube-state-metrics,app.kubernetes.io/version=2.10.1,helm.sh/chart=kube-state-metrics-5.15.3,pod-template-hash=8667fd9775,release=robusta\ndefault robusta-prometheus-node-exporter-4c4w5 1/1 Running 0 155d 172.31.24.130 ip-172-31-24-130.us-east-2.compute.internal app.kubernetes.io/component=metrics,app.kubernetes.io/instance=robusta,app.kubernetes.io/managed-by=Helm,app.kubernetes.io/name=prometheus-node-exporter,app.kubernetes.io/part-of=prometheus-node-exporter,app.kubernetes.io/version=1.7.0,controller-revision-hash=6dfc9596d4,helm.sh/chart=prometheus-node-exporter-4.24.0,jobLabel=node-exporter,pod-template-generation=29,release=robusta\ndefault robusta-prometheus-node-exporter-998h6 1/1 Running 0 155d 172.31.5.104 ip-172-31-5-104.us-east-2.compute.internal app.kubernetes.io/component=metrics,app.kubernetes.io/instance=robusta,app.kubernetes.io/managed-by=Helm,app.kubernetes.io/name=prometheus-node-exporter,app.kubernetes.io/part-of=prometheus-node-exporter,app.kubernetes.io/version=1.7.0,controller-revision-hash=6dfc9596d4,helm.sh/chart=prometheus-node-exporter-4.24.0,jobLabel=node-exporter,pod-template-generation=29,release=robusta\ndefault robusta-prometheus-node-exporter-hvnkn 1/1 Running 0 155d 172.31.47.239 ip-172-31-47-239.us-east-2.compute.internal app.kubernetes.io/component=metrics,app.kubernetes.io/instance=robusta,app.kubernetes.io/managed-by=Helm,app.kubernetes.io/name=prometheus-node-exporter,app.kubernetes.io/part-of=prometheus-node-exporter,app.kubernetes.io/version=1.7.0,controller-revision-hash=6dfc9596d4,helm.sh/chart=prometheus-node-exporter-4.24.0,jobLabel=node-exporter,pod-template-generation=29,release=robusta\ndefault robusta-runner-fbb77cfb8-z24xg 1/1 Running 0 8d 172.31.32.167 ip-172-31-47-239.us-east-2.compute.internal app=robusta-runner,pod-template-hash=fbb77cfb8,robustaComponent=runner\nkube-system aws-node-85q4x 2/2 Running 0 269d 172.31.5.104 ip-172-31-5-104.us-east-2.compute.internal app.kubernetes.io/instance=aws-vpc-cni,app.kubernetes.io/name=aws-node,controller-revision-hash=8567bc7fd8,k8s-app=aws-node,pod-template-generation=1\nkube-system aws-node-jmpdm 2/2 Running 0 269d 172.31.24.130 ip-172-31-24-130.us-east-2.compute.internal app.kubernetes.io/instance=aws-vpc-cni,app.kubernetes.io/name=aws-node,controller-revision-hash=8567bc7fd8,k8s-app=aws-node,pod-template-generation=1\nkube-system aws-node-vrm27 2/2 Running 0 269d 172.31.47.239 ip-172-31-47-239.us-east-2.compute.internal app.kubernetes.io/instance=aws-vpc-cni,app.kubernetes.io/name=aws-node,controller-revision-hash=8567bc7fd8,k8s-app=a", + "token_count": 5355, + "tool_call_id": "call_3jdRhOTW5h3yaNlkhpbyvLN6" + }, + { + "role": "assistant", + "content": "All pods are healthy. No pods are in an unhealthy state.", + "token_count": 20 + }, + { + "role": "user", + "content": "Can you check the pods?", + "token_count": 13 + }, + { + "role": "assistant", + "tool_calls": + [ + { + "id": "call_Ihc7RsKsSxlXzDUPhnh133XD", + "type": "function", + "function": + { + "name": "kubectl_get_by_kind_in_cluster", + "arguments": "{\"kind\": \"pod\"}" + } + }, + { + "id": "call_HJI22a9KPTpGvGCyvOPCRcUp", + "type": "function", + "function": + { + "name": "kubectl_memory_requests_all_namespaces", + "arguments": "{}" + } + } + ] + }, + { + "name": "kubectl_get_by_kind_in_cluster", + "role": "tool", + "content": "Params used for the tool call: {\"kind\": \"pod\"}. The tool call output follows on the next line.\nNAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES LABELS\ndefault alertmanager-robusta-kube-prometheus-st-alertmanager-0 2/2 Running 0 155d 172.31.40.168 ip-172-31-47-239.us-east-2.compute.internal alertmanager=robusta-kube-prometheus-st-alertmanager,app.kubernetes.io/instance=robusta-kube-prometheus-st-alertmanager,app.kubernetes.io/managed-by=prometheus-operator,app.kubernetes.io/name=alertmanager,app.kubernetes.io/version=0.26.0,apps.kubernetes.io/pod-index=0,controller-revision-hash=alertmanager-robusta-kube-prometheus-st-alertmanager-57cd7fb46f,statefulset.kubernetes.io/pod-name=alertmanager-robusta-kube-prometheus-st-alertmanager-0\ndefault curl-deployment-6c67b4656-nhjwj 1/1 Running 0 28h 172.31.12.12 ip-172-31-5-104.us-east-2.compute.internal app=curl-app,pod-template-hash=6c67b4656\ndefault datadog-agent-sqj8m 2/2 Running 0 46m 172.31.41.35 ip-172-31-47-239.us-east-2.compute.internal agent.datadoghq.com/component=agent,agent.datadoghq.com/name=datadog,agent.datadoghq.com/provider=,app.kubernetes.io/component=agent,app.kubernetes.io/instance=datadog-agent,app.kubernetes.io/managed-by=datadog-operator,app.kubernetes.io/name=datadog-agent-deployment,app.kubernetes.io/part-of=default-datadog,app.kubernetes.io/version=,controller-revision-hash=b5bff5c95,pod-template-generation=2\ndefault datadog-agent-tjhbs 2/2 Running 0 47m 172.31.20.130 ip-172-31-24-130.us-east-2.compute.internal agent.datadoghq.com/component=agent,agent.datadoghq.com/name=datadog,agent.datadoghq.com/provider=,app.kubernetes.io/component=agent,app.kubernetes.io/instance=datadog-agent,app.kubernetes.io/managed-by=datadog-operator,app.kubernetes.io/name=datadog-agent-deployment,app.kubernetes.io/part-of=default-datadog,app.kubernetes.io/version=,controller-revision-hash=b5bff5c95,pod-template-generation=2\ndefault datadog-agent-zzxns 2/2 Running 0 47m 172.31.0.245 ip-172-31-5-104.us-east-2.compute.internal agent.datadoghq.com/component=agent,agent.datadoghq.com/name=datadog,agent.datadoghq.com/provider=,app.kubernetes.io/component=agent,app.kubernetes.io/instance=datadog-agent,app.kubernetes.io/managed-by=datadog-operator,app.kubernetes.io/name=datadog-agent-deployment,app.kubernetes.io/part-of=default-datadog,app.kubernetes.io/version=,controller-revision-hash=b5bff5c95,pod-template-generation=2\ndefault datadog-cluster-agent-55d484796b-x6vl2 1/1 Running 0 69m 172.31.1.219 ip-172-31-5-104.us-east-2.compute.internal agent.datadoghq.com/component=cluster-agent,agent.datadoghq.com/name=datadog,app.kubernetes.io/component=cluster-agent,app.kubernetes.io/instance=datadog-cluster-agent,app.kubernetes.io/managed-by=datadog-operator,app.kubernetes.io/name=datadog-agent-deployment,app.kubernetes.io/part-of=default-datadog,app.kubernetes.io/version=,pod-template-hash=55d484796b\ndefault datadog-operator-6c9b79787f-bxhrk 1/1 Running 0 72m 172.31.10.101 ip-172-31-5-104.us-east-2.compute.internal app.kubernetes.io/instance=datadog-operator,app.kubernetes.io/name=datadog-operator,pod-template-hash=6c9b79787f\ndefault meme-deployment-74db7bc95c-dkdhk 1/1 Running 0 28h 172.31.12.146 ip-172-31-5-104.us-east-2.compute.internal app=meme-app,pod-template-hash=74db7bc95c\ndefault meme-deployment-74db7bc95c-vznd5 1/1 Running 0 28h 172.31.43.211 ip-172-31-47-239.us-east-2.compute.internal app=meme-app,pod-template-hash=74db7bc95c\ndefault prometheus-robusta-kube-prometheus-st-prometheus-0 2/2 Running 0 123d 172.31.3.100 ip-172-31-5-104.us-east-2.compute.internal app.kubernetes.io/instance=robusta-kube-prometheus-st-prometheus,app.kubernetes.io/managed-by=prometheus-operator,app.kubernetes.io/name=prometheus,app.kubernetes.io/version=2.48.1,apps.kubernetes.io/pod-index=0,controller-revision-hash=prometheus-robusta-kube-prometheus-st-prometheus-55d87c869b,operator.prometheus.io/name=robusta-kube-prometheus-st-prometheus,operator.prometheus.io/shard=0,prometheus=robusta-kube-prometheus-st-prometheus,statefulset.kubernetes.io/pod-name=prometheus-robusta-kube-prometheus-st-prometheus-0\ndefault robusta-forwarder-6759cbdf94-5q4qh 1/1 Running 0 36d 172.31.27.42 ip-172-31-24-130.us-east-2.compute.internal app=robusta-forwarder,pod-template-hash=6759cbdf94\ndefault robusta-grafana-8487fb665f-2l8ss 3/3 Running 0 149d 172.31.33.141 ip-172-31-47-239.us-east-2.compute.internal app.kubernetes.io/instance=robusta,app.kubernetes.io/name=grafana,pod-template-hash=8487fb665f\ndefault robusta-holmes-7799dc97f6-f5n86 1/1 Running 0 9m37s 172.31.23.25 ip-172-31-24-130.us-east-2.compute.internal app=holmes,pod-template-hash=7799dc97f6\ndefault robusta-kube-prometheus-st-operator-6885c8f675-25qfq 1/1 Running 0 179d 172.31.47.106 ip-172-31-47-239.us-east-2.compute.internal app.kubernetes.io/instance=robusta,app.kubernetes.io/managed-by=Helm,app.kubernetes.io/part-of=kube-prometheus-stack,app.kubernetes.io/version=55.7.0,app=kube-prometheus-stack-operator,chart=kube-prometheus-stack-55.7.0,heritage=Helm,pod-template-hash=6885c8f675,release=robusta\ndefault robusta-kube-state-metrics-8667fd9775-slv9m 1/1 Running 0 179d 172.31.8.207 ip-172-31-5-104.us-east-2.compute.internal app.kubernetes.io/component=metrics,app.kubernetes.io/instance=robusta,app.kubernetes.io/managed-by=Helm,app.kubernetes.io/name=kube-state-metrics,app.kubernetes.io/part-of=kube-state-metrics,app.kubernetes.io/version=2.10.1,helm.sh/chart=kube-state-metrics-5.15.3,pod-template-hash=8667fd9775,release=robusta\ndefault robusta-prometheus-node-exporter-4c4w5 1/1 Running 0 155d 172.31.24.130 ip-172-31-24-130.us-east-2.compute.internal app.kubernetes.io/component=metrics,app.kubernetes.io/instance=robusta,app.kubernetes.io/managed-by=Helm,app.kubernetes.io/name=prometheus-node-exporter,app.kubernetes.io/part-of=prometheus-node-exporter,app.kubernetes.io/version=1.7.0,controller-revision-hash=6dfc9596d4,helm.sh/chart=prometheus-node-exporter-4.24.0,jobLabel=node-exporter,pod-template-generation=29,release=robusta\ndefault robusta-prometheus-node-exporter-998h6 1/1 Running 0 155d 172.31.5.104 ip-172-31-5-104.us-east-2.compute.internal app.kubernetes.io/component=metrics,app.kubernetes.io/instance=robusta,app.kubernetes.io/managed-by=Helm,app.kubernetes.io/name=prometheus-node-exporter,app.kubernetes.io/part-of=prometheus-node-exporter,app.kubernetes.io/version=1.7.0,controller-revision-hash=6dfc9596d4,helm.sh/chart=prometheus-node-exporter-4.24.0,jobLabel=node-exporter,pod-template-generation=29,release=robusta\ndefault robusta-prometheus-node-exporter-hvnkn 1/1 Running 0 155d 172.31.47.239 ip-172-31-47-239.us-east-2.compute.internal app.kubernetes.io/component=metrics,app.kubernetes.io/instance=robusta,app.kubernetes.io/managed-by=Helm,app.kubernetes.io/name=prometheus-node-exporter,app.kubernetes.io/part-of=prometheus-node-exporter,app.kubernetes.io/version=1.7.0,controller-revision-hash=6dfc9596d4,helm.sh/chart=prometheus-node-exporter-4.24.0,jobLabel=node-exporter,pod-template-generation=29,release=robusta\ndefault robusta-runner-fbb77cfb8-z24xg 1/1 Running 0 8d 172.31.32.167 ip-172-31-47-239.us-east-2.compute.internal app=robusta-runner,pod-template-hash=fbb77cfb8,robustaComponent=runner\nkube-system aws-node-85q4x 2/2 Running 0 269d 172.31.5.104 ip-172-31-5-104.us-east-2.compute.internal app.kubernetes.io/instance=aws-vpc-cni,app.kubernetes.io/name=aws-node,controller-revision-hash=8567bc7fd8,k8s-app=aws-node,pod-template-generation=1\nkube-system aws-node-jmpdm 2/2 Running 0 269d 172.31.24.130 ip-172-31-24-130.us-east-2.compute.internal app.kubernetes.io/instance=aws-vpc-cni,app.kubernetes.io/name=aws-node,controller-revision-hash=8567bc7fd8,k8s-app=aws-node,pod-template-generation=1\nkube-system aws-node-vrm27 2/2 Running 0 269d 172.31.47.239 ip-172-31-47-239.us-east-2.compute.internal app.kubernetes.io/instance=aws-vpc-cni,app.kubernetes.io/name=aws-node,controller-revision-hash=8567bc7fd8,k8s-app=a", + "token_count": 5355, + "tool_call_id": "call_Ihc7RsKsSxlXzDUPhnh133XD" + }, + { + "name": "kubectl_memory_requests_all_namespaces", + "role": "tool", + "content": "default prometheus-robusta-kube-prometheus-st-prometheus-0 2048 Mi\ndefault robusta-runner-fbb77cfb8-z24xg 1024 Mi\ndefault robusta-holmes-7799dc97f6-f5n86 1024 Mi\ndefault robusta-forwarder-6759cbdf94-5q4qh 512 Mi\nsock-shop shipping-c4d77b86d-2z4jn 300 Mi\nsock-shop queue-master-566bfb8644-lml7l 300 Mi\nsock-shop orders-697cb4957d-l57cs 300 Mi\nsock-shop front-end-6bc5876d74-nml48 300 Mi\ndefault robusta-kube-state-metrics-8667fd9775-slv9m 256 Mi\nkube-system ebs-csi-controller-6cbc58f9b7-wbznn 240 Mi\nkube-system ebs-csi-controller-6cbc58f9b7-bdvd6 240 Mi\nsock-shop carts-f54496f6d-f8lf5 200 Mi\ndefault alertmanager-robusta-kube-prometheus-st-alertmanager-0 128 Mi\nkube-system ebs-csi-node-w7x6r 120 Mi\nkube-system ebs-csi-node-v9cc4 120 Mi\nkube-system ebs-csi-node-2cwvk 120 Mi\nsock-shop user-574f64957c-86ldd 100 Mi\nsock-shop payment-84bbbfd97f-tr828 100 Mi\nsock-shop catalogue-f7687cb4-zsngc 100 Mi\nkube-system coredns-748f7b95bf-fd6pp 70 Mi\nkube-system coredns-748f7b95bf-9dbdv 70 Mi\ndefault robusta-prometheus-node-exporter-hvnkn 50 Mi\ndefault robusta-prometheus-node-exporter-998h6 50 Mi\ndefault robusta-prometheus-node-exporter-4c4w5 50 Mi\ntest-ns nginx 0 Mi\ntest-ns client 0 Mi\nsock-shop user-db-7f98f68489-sq645 0 Mi\nsock-shop session-db-59cc4b855f-tt6dk 0 Mi\nsock-shop rabbitmq-556cb847-4j6jr 0 Mi\nsock-shop orders-db-658fc79675-58d24 0 Mi\nsock-shop catalogue-db-c948fd796-w4vzl 0 Mi\nsock-shop carts-db-676c6b5865-wmvqt 0 Mi\nkube-system kube-proxy-dfdd6 0 Mi\nkube-system kube-proxy-8bt9r 0 Mi\nkube-system kube-proxy-2gsrz 0 Mi\nkube-system aws-node-vrm27 0 Mi\nkube-system aws-node-jmpdm 0 Mi\nkube-system aws-node-85q4x 0 Mi\ndefault robusta-kube-prometheus-st-operator-6885c8f675-25qfq 0 Mi\ndefault robusta-grafana-8487fb665f-2l8ss 0 Mi\ndefault meme-deployment-74db7bc95c-vznd5 0 Mi\ndefault meme-deployment-74db7bc95c-dkdhk 0 Mi\ndefault datadog-operator-6c9b79787f-bxhrk 0 Mi\ndefault datadog-cluster-agent-55d484796b-x6vl2 0 Mi\ndefault datadog-agent-zzxns 0 Mi\ndefault datadog-agent-tjhbs 0 Mi\ndefault datadog-agent-sqj8m 0 Mi\ndefault curl-deployment-6c67b4656-nhjwj 0 Mi", + "token_count": 837, + "tool_call_id": "call_HJI22a9KPTpGvGCyvOPCRcUp" + }, + { + "role": "assistant", + "content": "All pods are running and healthy. Memory requests vary across pods, with some configured for high memory usage (e.g., `prometheus-robusta-kube-prometheus-st-prometheus-0` at 2048 Mi) and others at 0 Mi.", + "token_count": 60 + }, + { + "role": "user", + "content": "what is the issue?", + "token_count": 12 + }, + { + "role": "assistant", + "content": "No issues detected. All pods are running and healthy. Memory requests are configured appropriately for workloads.", + "token_count": 26 + }, + { + "role": "user", + "content": "Can you show the memory usage of pods?", + "token_count": 16 + }, + { + "role": "assistant", + "content": "Memory requests for pods:\n\n- `prometheus-robusta-kube-prometheus-st-prometheus-0`: 2048 Mi\n- `robusta-runner-fbb77cfb8-z24xg`: 1024 Mi\n- `robusta-holmes-7799dc97f6-f5n86`: 1024 Mi\n- `robusta-forwarder-6759cbdf94-5q4qh`: 512 Mi\n- `shipping-c4d77b86d-2z4jn`: 300 Mi\n- `queue-master-566bfb8644-lml7l`: 300 Mi\n- `orders-697cb4957d-l57cs`: 300 Mi\n- `front-end-6bc5876d74-nml48`: 300 Mi\n- `robusta-kube-state-metrics-8667fd9775-slv9m`: 256 Mi\n- `ebs-csi-controller-6cbc58f9b7-wbznn`: 240 Mi\n- `ebs-csi-controller-6cbc58f9b7-bdvd6`: 240 Mi\n- `carts-f54496f6d-f8lf5`: 200 Mi\n- `alertmanager-robusta-kube-prometheus-st-alertmanager-0`: 128 Mi\n- `ebs-csi-node-w7x6r`: 120 Mi\n- `ebs-csi-node-v9cc4`: 120 Mi\n- `ebs-csi-node-2cwvk`: 120 Mi\n- `user-574f64957c-86ldd`: 100 Mi\n- `payment-84bbbfd97f-tr828`: 100 Mi\n- `catalogue-f7687cb4-zsngc`: 100 Mi\n- `coredns-748f7b95bf-fd6pp`: 70 Mi\n- `coredns-748f7b95bf-9dbdv`: 70 Mi\n- `robusta-prometheus-node-exporter-hvnkn`: 50 Mi\n- `robusta-prometheus-node-exporter-998h6`: 50 Mi\n- `robusta-prometheus-node-exporter-4c4w5`: 50 Mi\n\nSeveral pods have 0 Mi memory requests, indicating no explicit memory allocation.", + "token_count": 501 + }, + { + "role": "user", + "content": "Can you get the memory usage of the catalogue pods?", + "token_count": 18 + }, + { + "role": "assistant", + "tool_calls": + [ + { + "id": "call_MKrt0Plp01VBqTHylPpBBeOv", + "type": "function", + "function": + { + "name": "kubectl_find_resource", + "arguments": "{\"kind\":\"pod\",\"keyword\":\"catalogue\"}" + } + } + ] + }, + { + "name": "kubectl_find_resource", + "role": "tool", + "content": "Params used for the tool call: {\"kind\": \"pod\", \"keyword\": \"catalogue\"}. The tool call output follows on the next line.\nsock-shop catalogue-db-c948fd796-w4vzl 1/1 Running 0 28h 172.31.37.165 ip-172-31-47-239.us-east-2.compute.internal name=catalogue-db,pod-template-hash=c948fd796\nsock-shop catalogue-f7687cb4-zsngc 1/1 Running 0 28h 172.31.17.128 ip-172-31-24-130.us-east-2.compute.internal name=catalogue,pod-template-hash=f7687cb4", + "token_count": 189, + "tool_call_id": "call_MKrt0Plp01VBqTHylPpBBeOv" + }, + { + "role": "assistant", + "tool_calls": + [ + { + "id": "call_PD0PiIonbVxBKoL5zN1mXTOY", + "type": "function", + "function": + { + "name": "kubectl_describe", + "arguments": "{\"kind\": \"pod\", \"name\": \"catalogue-f7687cb4-zsngc\", \"namespace\": \"sock-shop\"}" + } + }, + { + "id": "call_xA1KBGBe0GHDnPYRv4Md67wr", + "type": "function", + "function": + { + "name": "kubectl_describe", + "arguments": "{\"kind\": \"pod\", \"name\": \"catalogue-db-c948fd796-w4vzl\", \"namespace\": \"sock-shop\"}" + } + } + ] + }, + { + "name": "kubectl_describe", + "role": "tool", + "content": "Params used for the tool call: {\"kind\": \"pod\", \"name\": \"catalogue-db-c948fd796-w4vzl\", \"namespace\": \"sock-shop\"}. The tool call output follows on the next line.\nName: catalogue-db-c948fd796-w4vzl\nNamespace: sock-shop\nPriority: 0\nService Account: default\nNode: ip-172-31-47-239.us-east-2.compute.internal/172.31.47.239\nStart Time: Wed, 30 Jul 2025 07:19:27 +0000\nLabels: name=catalogue-db\n pod-template-hash=c948fd796\nAnnotations: \nStatus: Running\nIP: 172.31.37.165\nIPs:\n IP: 172.31.37.165\nControlled By: ReplicaSet/catalogue-db-c948fd796\nContainers:\n catalogue-db:\n Container ID: containerd://c350ec56066bf52557a10212e6e4279d9c504add2d17c87de03872ba22393abd\n Image: weaveworksdemos/catalogue-db:0.3.0\n Image ID: docker.io/weaveworksdemos/catalogue-db@sha256:7ba74ec9adf88f6625b8d85d3323d1ee5232b39877e1590021ea485cf9457251\n Port: 3306/TCP\n Host Port: 0/TCP\n State: Running\n Started: Wed, 30 Jul 2025 07:19:29 +0000\n Ready: True\n Restart Count: 0\n Environment:\n MYSQL_ROOT_PASSWORD: fake_password\n MYSQL_DATABASE: socksdb\n Mounts:\n /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-glnbd (ro)\nConditions:\n Type Status\n PodReadyToStartContainers True \n Initialized True \n Ready True \n ContainersReady True \n PodScheduled True \nVolumes:\n kube-api-access-glnbd:\n Type: Projected (a volume that contains injected data from multiple sources)\n TokenExpirationSeconds: 3607\n ConfigMapName: kube-root-ca.crt\n ConfigMapOptional: \n DownwardAPI: true\nQoS Class: BestEffort\nNode-Selectors: beta.kubernetes.io/os=linux\nTolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s\n node.kubernetes.io/unreachable:NoExecute op=Exists for 300s\nEvents: ", + "token_count": 607, + "tool_call_id": "call_xA1KBGBe0GHDnPYRv4Md67wr" + }, + { + "name": "kubectl_describe", + "role": "tool", + "content": "Params used for the tool call: {\"kind\": \"pod\", \"name\": \"catalogue-f7687cb4-zsngc\", \"namespace\": \"sock-shop\"}. The tool call output follows on the next line.\nName: catalogue-f7687cb4-zsngc\nNamespace: sock-shop\nPriority: 0\nService Account: default\nNode: ip-172-31-24-130.us-east-2.compute.internal/172.31.24.130\nStart Time: Wed, 30 Jul 2025 07:19:27 +0000\nLabels: name=catalogue\n pod-template-hash=f7687cb4\nAnnotations: \nStatus: Running\nIP: 172.31.17.128\nIPs:\n IP: 172.31.17.128\nControlled By: ReplicaSet/catalogue-f7687cb4\nContainers:\n catalogue:\n Container ID: containerd://2257500d367d541b8bb518cc74965b98d51c532bf3227c380b6456c6bb788822\n Image: weaveworksdemos/catalogue:0.3.5\n Image ID: docker.io/weaveworksdemos/catalogue@sha256:0147a65b7116569439eefb1a6dbed455fe022464ef70e0c3cab75bc4a226b39b\n Port: 80/TCP\n Host Port: 0/TCP\n Command:\n /app\n Args:\n -port=80\n State: Running\n Started: Wed, 30 Jul 2025 07:19:43 +0000\n Ready: True\n Restart Count: 0\n Limits:\n cpu: 200m\n memory: 200Mi\n Requests:\n cpu: 100m\n memory: 100Mi\n Liveness: http-get http://:80/health delay=300s timeout=1s period=3s #success=1 #failure=3\n Readiness: http-get http://:80/health delay=180s timeout=1s period=3s #success=1 #failure=3\n Environment: \n Mounts:\n /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-675v5 (ro)\nConditions:\n Type Status\n PodReadyToStartContainers True \n Initialized True \n Ready True \n ContainersReady True \n PodScheduled True \nVolumes:\n kube-api-access-675v5:\n Type: Projected (a volume that contains injected data from multiple sources)\n TokenExpirationSeconds: 3607\n ConfigMapName: kube-root-ca.crt\n ConfigMapOptional: \n DownwardAPI: true\nQoS Class: Burstable\nNode-Selectors: beta.kubernetes.io/os=linux\nTolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s\n node.kubernetes.io/unreachable:NoExecute op=Exists for 300s\nEvents: ", + "token_count": 712, + "tool_call_id": "call_PD0PiIonbVxBKoL5zN1mXTOY" + }, + { + "role": "assistant", + "content": "Memory usage details for catalogue pods:\n\n1. `catalogue-f7687cb4-zsngc`:\n - Memory Requests: 100 Mi\n - Memory Limits: 200 Mi\n\n2. `catalogue-db-c948fd796-w4vzl`:\n - No explicit memory requests or limits configured.", + "token_count": 74 + }, + { + "role": "user", + "content": "can you hosw this as graph?", + "token_count": 15 + }, + { + "role": "assistant", + "tool_calls": + [ + { + "id": "call_kcWueR4KHPUetfvnIMNXShqW", + "type": "function", + "function": + { + "name": "list_active_datadog_metrics", + "arguments": "{\"tag_filter\":\"name:catalogue\"}" + } + } + ] + }, + { + "name": "list_active_datadog_metrics", + "role": "tool", + "content": "Params used for the tool call: {\"tag_filter\": \"name:catalogue\"}. The tool call output follows on the next line.\nMetric Name\n--------------------------------------------------", + "token_count": 40, + "tool_call_id": "call_kcWueR4KHPUetfvnIMNXShqW" + }, + { + "role": "assistant", + "content": "No active memory metrics found for the catalogue pods. Unable to generate a graph.", + "token_count": 23 + }, + { + "role": "user", + "content": "use datadog metrics", + "token_count": 12 + }, + { + "role": "assistant", + "tool_calls": + [ + { + "id": "call_FXT5lOJEexS8Cc8Bc0ZsLI3b", + "type": "function", + "function": + { + "name": "query_datadog_metrics", + "arguments": "{\"query\":\"kubernetes.memory.usage{pod:catalogue-f7687cb4-zsngc}\",\"from_time\":\"-3600\",\"to_time\":\"0\"}" + } + } + ] + }, + { + "name": "query_datadog_metrics", + "role": "tool", + "content": "Params used for the tool call: {\"query\": \"kubernetes.memory.usage{pod:catalogue-f7687cb4-zsngc}\", \"from_time\": \"-3600\", \"to_time\": \"0\"}. The tool call output follows on the next line.\nnull", + "token_count": 65, + "tool_call_id": "call_FXT5lOJEexS8Cc8Bc0ZsLI3b" + }, + { + "role": "assistant", + "tool_calls": + [ + { + "id": "call_5seNGFHe42oHfldxm1UXpOpb", + "type": "function", + "function": + { + "name": "query_datadog_metrics", + "arguments": "{\"query\":\"kubernetes.memory.usage{pod:catalogue-db-c948fd796-w4vzl}\",\"from_time\":\"-3600\",\"to_time\":\"0\"}" + } + } + ] + }, + { + "name": "query_datadog_metrics", + "role": "tool", + "content": "Params used for the tool call: {\"query\": \"kubernetes.memory.usage{pod:catalogue-db-c948fd796-w4vzl}\", \"from_time\": \"-3600\", \"to_time\": \"0\"}. The tool call output follows on the next line.\nnull", + "token_count": 65, + "tool_call_id": "call_5seNGFHe42oHfldxm1UXpOpb" + }, + { + "role": "assistant", + "content": "No memory usage data available for the catalogue pods in Datadog metrics. Unable to generate a graph." + } +] diff --git a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml new file mode 100644 index 0000000000..8a18b7c10d --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml @@ -0,0 +1,15 @@ +description: | + A test case that simulates an issue real customer bumped into. + When holmes were asked to fetch logs, they were not able to do so. + The main reason behind this are do to the conversation history and the lack of explicity saying that we enabled datadog logs toolset as part of the system prompt logging section. + +user_question: "Can you retrieve the datadog logs for this?" + +expected_output: | + Logs for catalogue-f7687cb4-zsngc pod show consistent health checks with no errors. + +evaluation: + correctness: 1 + +tags: + - easy diff --git a/tests/llm/utils/test_case_utils.py b/tests/llm/utils/test_case_utils.py index 1bb5bd273b..aca9dda14f 100644 --- a/tests/llm/utils/test_case_utils.py +++ b/tests/llm/utils/test_case_utils.py @@ -113,7 +113,7 @@ def load_investigate_test_cases(self) -> List[InvestigateTestCase]: return cast(List[InvestigateTestCase], self.load_test_cases()) def load_ask_holmes_test_cases(self) -> List[AskHolmesTestCase]: - return cast(List[AskHolmesTestCase], self.load_test_cases()) + return self.load_test_cases() def load_test_cases(self) -> List[HolmesTestCase]: test_cases: List[HolmesTestCase] = [] From b77599d9e5260004e81fd1742d755eb0dbc802bd Mon Sep 17 00:00:00 2001 From: Mohse Morad Date: Sun, 3 Aug 2025 15:10:42 +0300 Subject: [PATCH 02/11] Add ask 91 eval --- poetry.lock | 17 +++++++++++- pyproject.toml | 1 + .../91_calling_datadog/test_case.yaml | 2 +- .../91_calling_datadog/toolsets.yaml | 27 +++++++++++++++++++ 4 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 tests/llm/fixtures/test_ask_holmes/91_calling_datadog/toolsets.yaml diff --git a/poetry.lock b/poetry.lock index 9342c7e1fc..b66c85047d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3225,6 +3225,21 @@ pytest = ">=6.2.5" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] +[[package]] +name = "pytest-dotenv" +version = "0.5.2" +description = "A py.test plugin that parses environment files before running tests" +optional = false +python-versions = "*" +files = [ + {file = "pytest-dotenv-0.5.2.tar.gz", hash = "sha256:2dc6c3ac6d8764c71c6d2804e902d0ff810fa19692e95fe138aefc9b1aa73732"}, + {file = "pytest_dotenv-0.5.2-py3-none-any.whl", hash = "sha256:40a2cece120a213898afaa5407673f6bd924b1fa7eafce6bda0e8abffe2f710f"}, +] + +[package.dependencies] +pytest = ">=5.0.0" +python-dotenv = ">=0.9.1" + [[package]] name = "pytest-shared-session-scope" version = "0.4.0" @@ -4720,4 +4735,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "998994ecb29e136530456a098bf5b0448a3dbe41bcf07aad047fcfbf32c06013" +content-hash = "cfb3a97e3bcba6cab324468ef3a91616c198b4e51d8276b2cbfd89d360d1c70b" diff --git a/pyproject.toml b/pyproject.toml index d80735b7f0..91ec625637 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ freezegun = "^1.5.1" tomli = {version = "^2.0.1", python = "<3.11"} mypy = "^1.16.0" pytest-cov = "^6.2.1" +pytest-dotenv = "^0.5.2" [build-system] requires = ["poetry-core"] diff --git a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml index 8a18b7c10d..a935f061b0 100644 --- a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml @@ -3,7 +3,7 @@ description: | When holmes were asked to fetch logs, they were not able to do so. The main reason behind this are do to the conversation history and the lack of explicity saying that we enabled datadog logs toolset as part of the system prompt logging section. -user_question: "Can you retrieve the datadog logs for this?" +user_prompt: "Can you retrieve the datadog logs for this?" expected_output: | Logs for catalogue-f7687cb4-zsngc pod show consistent health checks with no errors. diff --git a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/toolsets.yaml new file mode 100644 index 0000000000..40e92b7eb1 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/toolsets.yaml @@ -0,0 +1,27 @@ +toolsets: + prometheus/metrics: + enabled: False + kubernetes/kube-lineage-extras: + enabled: true + kubernetes/logs: + enabled: False + kubernetes/core: + enabled: true + datadog/logs: + enabled: True + config: + dd_app_key: "{{ env.DATADOG_APP_KEY }}" + dd_api_key: "{{ env.DATADOG_API_KEY }}" + site_api_url: https://api.datadoghq.eu + datadog/metrics: + enabled: True + config: + dd_app_key: "{{ env.DATADOG_APP_KEY }}" + dd_api_key: "{{ env.DATADOG_API_KEY }}" + site_api_url: https://api.datadoghq.eu + datadog/traces: + enabled: True + config: + dd_app_key: "{{ env.DATADOG_APP_KEY }}" + dd_api_key: "{{ env.DATADOG_API_KEY }}" + site_api_url: https://api.datadoghq.eu From 5196a3754be8eaac78d4e813386db4fe3defd1ac Mon Sep 17 00:00:00 2001 From: Mohse Morad Date: Mon, 4 Aug 2025 15:45:53 +0300 Subject: [PATCH 03/11] Fix tool mock generation --- holmes/core/tools_utils/toolset_utils.py | 9 +- tests/llm/conftest.py | 16 +- .../test_case.yaml | 2 +- ...hop_catalogue-db-c948fd796-w4vzl_-3600.txt | 2 + ...ck-shop_catalogue-f7687cb4-zsngc_-3600.txt | 1002 +++++++++++++++++ .../91_calling_datadog/test_case.yaml | 3 + tests/llm/utils/mock_toolset.py | 228 ++-- tests/llm/utils/test_case_utils.py | 13 + 8 files changed, 1156 insertions(+), 119 deletions(-) create mode 100644 tests/llm/fixtures/test_ask_holmes/91_calling_datadog/fetch_pod_logssock-shop_catalogue-db-c948fd796-w4vzl_-3600.txt create mode 100644 tests/llm/fixtures/test_ask_holmes/91_calling_datadog/fetch_pod_logssock-shop_catalogue-f7687cb4-zsngc_-3600.txt diff --git a/holmes/core/tools_utils/toolset_utils.py b/holmes/core/tools_utils/toolset_utils.py index 277d89d8f9..38bd67851c 100644 --- a/holmes/core/tools_utils/toolset_utils.py +++ b/holmes/core/tools_utils/toolset_utils.py @@ -16,12 +16,17 @@ def filter_out_default_logging_toolset(toolsets: list[Toolset]) -> list[Toolset] All other types of toolsets are included as is. """ - logging_toolsets: list[BasePodLoggingToolset] = [] + logging_toolsets: list[Toolset] = [] final_toolsets: list[Toolset] = [] for ts in toolsets: + toolset_type = ( + ts.original_toolset_type + if hasattr(ts, "original_toolset_type") + else type(ts) + ) if ( - isinstance(ts, BasePodLoggingToolset) + issubclass(toolset_type, BasePodLoggingToolset) and ts.status == ToolsetStatusEnum.ENABLED ): logging_toolsets.append(ts) diff --git a/tests/llm/conftest.py b/tests/llm/conftest.py index d2191af0a8..1bb5eba8a6 100644 --- a/tests/llm/conftest.py +++ b/tests/llm/conftest.py @@ -53,11 +53,21 @@ def mock_generation_config(request): if regenerate_all_mocks: generate_mocks = True + run_live = os.getenv("RUN_LIVE", "False").lower() in ("true", "1", "t") + if generate_mocks and not run_live: + print( + "⚠️ WARNING: --generate-mocks is set but RUN_LIVE is not set. This will not generate mocks." + ) + pytest.skip( + "Skipping test case because --generate-mocks is set but RUN_LIVE is not set." + ) + # Determine mode based on environment and options - if os.getenv("RUN_LIVE", "False").lower() in ("true", "1", "t"): + + if generate_mocks: + mode = MockMode.GENERATE # live & generate + elif run_live: mode = MockMode.LIVE - elif generate_mocks: - mode = MockMode.GENERATE else: mode = MockMode.MOCK diff --git a/tests/llm/fixtures/test_ask_holmes/88_affinity_like_but_taints/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/88_affinity_like_but_taints/test_case.yaml index 8a1f49d6d2..083759cb80 100644 --- a/tests/llm/fixtures/test_ask_holmes/88_affinity_like_but_taints/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/88_affinity_like_but_taints/test_case.yaml @@ -4,7 +4,7 @@ description: | and the actual issue is missing tolerations for node taints, not zone constraints. Tests that LLM identifies the real scheduling constraint. -user_question: "Whats wrong with database-primary deployment in production-88 isn't starting?" +user_prompt: "Whats wrong with database-primary deployment in production-88 isn't starting?" # CodeRabbit: RWO PVC + 4 replicas ⇒ inevitable Multi-Attach errors # A single ReadWriteOnce volume cannot be mounted by four pods on different nodes. This scheduling error will surface before the intended taint/toleration failure, potentially breaking the test signal. diff --git a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/fetch_pod_logssock-shop_catalogue-db-c948fd796-w4vzl_-3600.txt b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/fetch_pod_logssock-shop_catalogue-db-c948fd796-w4vzl_-3600.txt new file mode 100644 index 0000000000..92088486e6 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/fetch_pod_logssock-shop_catalogue-db-c948fd796-w4vzl_-3600.txt @@ -0,0 +1,2 @@ +{"toolset_name":"datadog/logs","tool_name":"fetch_pod_logs","match_params":{"pod_name":"catalogue-db-c948fd796-w4vzl","namespace":"sock-shop","start_time":"-3600"}} +{"schema_version": "robusta:v1.0.0", "status": "no_data", "error": null, "return_code": null, "url": null, "invocation": null, "params": {"namespace": "sock-shop", "pod_name": "catalogue-db-c948fd796-w4vzl", "start_time": "-3600", "end_time": null, "filter": null, "limit": null}} diff --git a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/fetch_pod_logssock-shop_catalogue-f7687cb4-zsngc_-3600.txt b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/fetch_pod_logssock-shop_catalogue-f7687cb4-zsngc_-3600.txt new file mode 100644 index 0000000000..2069e06cb3 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/fetch_pod_logssock-shop_catalogue-f7687cb4-zsngc_-3600.txt @@ -0,0 +1,1002 @@ +{"toolset_name":"datadog/logs","tool_name":"fetch_pod_logs","match_params":{"pod_name":"catalogue-f7687cb4-zsngc","namespace":"sock-shop","start_time":"-3600"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": null, "url": null, "invocation": null, "params": {"namespace": "sock-shop", "pod_name": "catalogue-f7687cb4-zsngc", "start_time": "-3600", "end_time": null, "filter": null, "limit": null}} +ts=2025-08-04T11:51:00Z caller=logging.go:85 method=Health result=2 took=40µs +ts=2025-08-04T11:51:00Z caller=logging.go:85 method=Health result=2 took=25.103µs +ts=2025-08-04T11:51:03Z caller=logging.go:85 method=Health result=2 took=41.358µs +ts=2025-08-04T11:51:03Z caller=logging.go:85 method=Health result=2 took=39.437µs +ts=2025-08-04T11:51:06Z caller=logging.go:85 method=Health result=2 took=41.145µs +ts=2025-08-04T11:51:06Z caller=logging.go:85 method=Health result=2 took=41.144µs +ts=2025-08-04T11:51:09Z caller=logging.go:85 method=Health result=2 took=30.161µs +ts=2025-08-04T11:51:09Z caller=logging.go:85 method=Health result=2 took=25.326µs +ts=2025-08-04T11:51:12Z caller=logging.go:85 method=Health result=2 took=32.356µs +ts=2025-08-04T11:51:12Z caller=logging.go:85 method=Health result=2 took=27.368µs +ts=2025-08-04T11:51:15Z caller=logging.go:85 method=Health result=2 took=33.632µs +ts=2025-08-04T11:51:15Z caller=logging.go:85 method=Health result=2 took=25.356µs +ts=2025-08-04T11:51:18Z caller=logging.go:85 method=Health result=2 took=32.285µs +ts=2025-08-04T11:51:18Z caller=logging.go:85 method=Health result=2 took=25.204µs +ts=2025-08-04T11:51:21Z caller=logging.go:85 method=Health result=2 took=30.697µs +ts=2025-08-04T11:51:21Z caller=logging.go:85 method=Health result=2 took=132.986µs +ts=2025-08-04T11:51:24Z caller=logging.go:85 method=Health result=2 took=30.441µs +ts=2025-08-04T11:51:24Z caller=logging.go:85 method=Health result=2 took=29.565µs +ts=2025-08-04T11:51:27Z caller=logging.go:85 method=Health result=2 took=41.226µs +ts=2025-08-04T11:51:27Z caller=logging.go:85 method=Health result=2 took=41.221µs +ts=2025-08-04T11:51:30Z caller=logging.go:85 method=Health result=2 took=32.532µs +ts=2025-08-04T11:51:30Z caller=logging.go:85 method=Health result=2 took=25.081µs +ts=2025-08-04T11:51:33Z caller=logging.go:85 method=Health result=2 took=30.765µs +ts=2025-08-04T11:51:33Z caller=logging.go:85 method=Health result=2 took=25.087µs +ts=2025-08-04T11:51:36Z caller=logging.go:85 method=Health result=2 took=32.976µs +ts=2025-08-04T11:51:36Z caller=logging.go:85 method=Health result=2 took=25.153µs +ts=2025-08-04T11:51:39Z caller=logging.go:85 method=Health result=2 took=42.439µs +ts=2025-08-04T11:51:39Z caller=logging.go:85 method=Health result=2 took=27.787µs +ts=2025-08-04T11:51:42Z caller=logging.go:85 method=Health result=2 took=44.89µs +ts=2025-08-04T11:51:42Z caller=logging.go:85 method=Health result=2 took=30.114µs +ts=2025-08-04T11:51:45Z caller=logging.go:85 method=Health result=2 took=32.496µs +ts=2025-08-04T11:51:45Z caller=logging.go:85 method=Health result=2 took=25.214µs +ts=2025-08-04T11:51:48Z caller=logging.go:85 method=Health result=2 took=41.14µs +ts=2025-08-04T11:51:48Z caller=logging.go:85 method=Health result=2 took=35.509µs +ts=2025-08-04T11:51:51Z caller=logging.go:85 method=Health result=2 took=32.461µs +ts=2025-08-04T11:51:51Z caller=logging.go:85 method=Health result=2 took=26.077µs +ts=2025-08-04T11:51:54Z caller=logging.go:85 method=Health result=2 took=31.807µs +ts=2025-08-04T11:51:54Z caller=logging.go:85 method=Health result=2 took=15.469µs +ts=2025-08-04T11:51:57Z caller=logging.go:85 method=Health result=2 took=31.315µs +ts=2025-08-04T11:51:57Z caller=logging.go:85 method=Health result=2 took=25.335µs +ts=2025-08-04T11:52:00Z caller=logging.go:85 method=Health result=2 took=79.382µs +ts=2025-08-04T11:52:00Z caller=logging.go:85 method=Health result=2 took=40.458µs +ts=2025-08-04T11:52:03Z caller=logging.go:85 method=Health result=2 took=33.885µs +ts=2025-08-04T11:52:03Z caller=logging.go:85 method=Health result=2 took=51.203µs +ts=2025-08-04T11:52:06Z caller=logging.go:85 method=Health result=2 took=31.65µs +ts=2025-08-04T11:52:06Z caller=logging.go:85 method=Health result=2 took=26.073µs +ts=2025-08-04T11:52:09Z caller=logging.go:85 method=Health result=2 took=31.016µs +ts=2025-08-04T11:52:09Z caller=logging.go:85 method=Health result=2 took=25.096µs +ts=2025-08-04T11:52:12Z caller=logging.go:85 method=Health result=2 took=33.795µs +ts=2025-08-04T11:52:12Z caller=logging.go:85 method=Health result=2 took=25.264µs +ts=2025-08-04T11:52:15Z caller=logging.go:85 method=Health result=2 took=31.837µs +ts=2025-08-04T11:52:15Z caller=logging.go:85 method=Health result=2 took=25.009µs +ts=2025-08-04T11:52:18Z caller=logging.go:85 method=Health result=2 took=41.857µs +ts=2025-08-04T11:52:18Z caller=logging.go:85 method=Health result=2 took=58.152µs +ts=2025-08-04T11:52:21Z caller=logging.go:85 method=Health result=2 took=32.149µs +ts=2025-08-04T11:52:21Z caller=logging.go:85 method=Health result=2 took=25.47µs +ts=2025-08-04T11:52:24Z caller=logging.go:85 method=Health result=2 took=32.085µs +ts=2025-08-04T11:52:24Z caller=logging.go:85 method=Health result=2 took=40.236µs +ts=2025-08-04T11:52:27Z caller=logging.go:85 method=Health result=2 took=31.203µs +ts=2025-08-04T11:52:27Z caller=logging.go:85 method=Health result=2 took=38.517µs +ts=2025-08-04T11:52:30Z caller=logging.go:85 method=Health result=2 took=32.477µs +ts=2025-08-04T11:52:30Z caller=logging.go:85 method=Health result=2 took=25.367µs +ts=2025-08-04T11:52:33Z caller=logging.go:85 method=Health result=2 took=49.435µs +ts=2025-08-04T11:52:33Z caller=logging.go:85 method=Health result=2 took=48.162µs +ts=2025-08-04T11:52:36Z caller=logging.go:85 method=Health result=2 took=32.777µs +ts=2025-08-04T11:52:36Z caller=logging.go:85 method=Health result=2 took=24.965µs +ts=2025-08-04T11:52:39Z caller=logging.go:85 method=Health result=2 took=32.37µs +ts=2025-08-04T11:52:39Z caller=logging.go:85 method=Health result=2 took=30.643µs +ts=2025-08-04T11:52:42Z caller=logging.go:85 method=Health result=2 took=32.863µs +ts=2025-08-04T11:52:42Z caller=logging.go:85 method=Health result=2 took=25.516µs +ts=2025-08-04T11:52:45Z caller=logging.go:85 method=Health result=2 took=32.276µs +ts=2025-08-04T11:52:45Z caller=logging.go:85 method=Health result=2 took=27.028µs +ts=2025-08-04T11:52:48Z caller=logging.go:85 method=Health result=2 took=44.71µs +ts=2025-08-04T11:52:48Z caller=logging.go:85 method=Health result=2 took=35.271µs +ts=2025-08-04T11:52:51Z caller=logging.go:85 method=Health result=2 took=32.46µs +ts=2025-08-04T11:52:51Z caller=logging.go:85 method=Health result=2 took=25.656µs +ts=2025-08-04T11:52:54Z caller=logging.go:85 method=Health result=2 took=54.249µs +ts=2025-08-04T11:52:54Z caller=logging.go:85 method=Health result=2 took=38.05µs +ts=2025-08-04T11:52:57Z caller=logging.go:85 method=Health result=2 took=32.628µs +ts=2025-08-04T11:52:57Z caller=logging.go:85 method=Health result=2 took=25.283µs +ts=2025-08-04T11:53:00Z caller=logging.go:85 method=Health result=2 took=31.284µs +ts=2025-08-04T11:53:00Z caller=logging.go:85 method=Health result=2 took=26.226µs +ts=2025-08-04T11:53:03Z caller=logging.go:85 method=Health result=2 took=41.648µs +ts=2025-08-04T11:53:03Z caller=logging.go:85 method=Health result=2 took=37.497µs +ts=2025-08-04T11:53:06Z caller=logging.go:85 method=Health result=2 took=32.898µs +ts=2025-08-04T11:53:06Z caller=logging.go:85 method=Health result=2 took=25.541µs +ts=2025-08-04T11:53:09Z caller=logging.go:85 method=Health result=2 took=33.357µs +ts=2025-08-04T11:53:09Z caller=logging.go:85 method=Health result=2 took=25.909µs +ts=2025-08-04T11:53:12Z caller=logging.go:85 method=Health result=2 took=41.159µs +ts=2025-08-04T11:53:12Z caller=logging.go:85 method=Health result=2 took=39.512µs +ts=2025-08-04T11:53:15Z caller=logging.go:85 method=Health result=2 took=40.297µs +ts=2025-08-04T11:53:15Z caller=logging.go:85 method=Health result=2 took=35.428µs +ts=2025-08-04T11:53:18Z caller=logging.go:85 method=Health result=2 took=32.317µs +ts=2025-08-04T11:53:18Z caller=logging.go:85 method=Health result=2 took=25.475µs +ts=2025-08-04T11:53:21Z caller=logging.go:85 method=Health result=2 took=31.661µs +ts=2025-08-04T11:53:21Z caller=logging.go:85 method=Health result=2 took=23.051µs +ts=2025-08-04T11:53:24Z caller=logging.go:85 method=Health result=2 took=44.603µs +ts=2025-08-04T11:53:24Z caller=logging.go:85 method=Health result=2 took=29.184µs +ts=2025-08-04T11:53:27Z caller=logging.go:85 method=Health result=2 took=31.988µs +ts=2025-08-04T11:53:27Z caller=logging.go:85 method=Health result=2 took=25.359µs +ts=2025-08-04T11:53:30Z caller=logging.go:85 method=Health result=2 took=33.741µs +ts=2025-08-04T11:53:30Z caller=logging.go:85 method=Health result=2 took=25.505µs +ts=2025-08-04T11:53:33Z caller=logging.go:85 method=Health result=2 took=32.093µs +ts=2025-08-04T11:53:33Z caller=logging.go:85 method=Health result=2 took=25.406µs +ts=2025-08-04T11:53:36Z caller=logging.go:85 method=Health result=2 took=41.47µs +ts=2025-08-04T11:53:36Z caller=logging.go:85 method=Health result=2 took=25.315µs +ts=2025-08-04T11:53:39Z caller=logging.go:85 method=Health result=2 took=42.872µs +ts=2025-08-04T11:53:39Z caller=logging.go:85 method=Health result=2 took=39.185µs +ts=2025-08-04T11:53:42Z caller=logging.go:85 method=Health result=2 took=41.454µs +ts=2025-08-04T11:53:42Z caller=logging.go:85 method=Health result=2 took=49.122µs +ts=2025-08-04T11:53:45Z caller=logging.go:85 method=Health result=2 took=31.112µs +ts=2025-08-04T11:53:45Z caller=logging.go:85 method=Health result=2 took=155.134µs +ts=2025-08-04T11:53:48Z caller=logging.go:85 method=Health result=2 took=31.531µs +ts=2025-08-04T11:53:48Z caller=logging.go:85 method=Health result=2 took=40.131µs +ts=2025-08-04T11:53:51Z caller=logging.go:85 method=Health result=2 took=31.765µs +ts=2025-08-04T11:53:51Z caller=logging.go:85 method=Health result=2 took=25.088µs +ts=2025-08-04T11:53:54Z caller=logging.go:85 method=Health result=2 took=32.189µs +ts=2025-08-04T11:53:54Z caller=logging.go:85 method=Health result=2 took=43.428µs +ts=2025-08-04T11:53:57Z caller=logging.go:85 method=Health result=2 took=31.102µs +ts=2025-08-04T11:53:57Z caller=logging.go:85 method=Health result=2 took=25.771µs +ts=2025-08-04T11:54:00Z caller=logging.go:85 method=Health result=2 took=32.443µs +ts=2025-08-04T11:54:00Z caller=logging.go:85 method=Health result=2 took=36.348µs +ts=2025-08-04T11:54:03Z caller=logging.go:85 method=Health result=2 took=31.816µs +ts=2025-08-04T11:54:03Z caller=logging.go:85 method=Health result=2 took=25.102µs +ts=2025-08-04T11:54:06Z caller=logging.go:85 method=Health result=2 took=31.635µs +ts=2025-08-04T11:54:06Z caller=logging.go:85 method=Health result=2 took=28.815µs +ts=2025-08-04T11:54:09Z caller=logging.go:85 method=Health result=2 took=41.59µs +ts=2025-08-04T11:54:09Z caller=logging.go:85 method=Health result=2 took=33.658µs +ts=2025-08-04T11:54:12Z caller=logging.go:85 method=Health result=2 took=32.124µs +ts=2025-08-04T11:54:12Z caller=logging.go:85 method=Health result=2 took=25.644µs +ts=2025-08-04T11:54:15Z caller=logging.go:85 method=Health result=2 took=31.692µs +ts=2025-08-04T11:54:15Z caller=logging.go:85 method=Health result=2 took=34.228µs +ts=2025-08-04T11:54:18Z caller=logging.go:85 method=Health result=2 took=31.097µs +ts=2025-08-04T11:54:18Z caller=logging.go:85 method=Health result=2 took=24.947µs +ts=2025-08-04T11:54:21Z caller=logging.go:85 method=Health result=2 took=42.858µs +ts=2025-08-04T11:54:21Z caller=logging.go:85 method=Health result=2 took=25.29µs +ts=2025-08-04T11:54:24Z caller=logging.go:85 method=Health result=2 took=40.321µs +ts=2025-08-04T11:54:24Z caller=logging.go:85 method=Health result=2 took=32.348µs +ts=2025-08-04T11:54:27Z caller=logging.go:85 method=Health result=2 took=32.311µs +ts=2025-08-04T11:54:27Z caller=logging.go:85 method=Health result=2 took=25.236µs +ts=2025-08-04T11:54:30Z caller=logging.go:85 method=Health result=2 took=33.461µs +ts=2025-08-04T11:54:30Z caller=logging.go:85 method=Health result=2 took=24.798µs +ts=2025-08-04T11:54:33Z caller=logging.go:85 method=Health result=2 took=30.959µs +ts=2025-08-04T11:54:33Z caller=logging.go:85 method=Health result=2 took=26.194µs +ts=2025-08-04T11:54:36Z caller=logging.go:85 method=Health result=2 took=31.992µs +ts=2025-08-04T11:54:36Z caller=logging.go:85 method=Health result=2 took=25.548µs +ts=2025-08-04T11:54:39Z caller=logging.go:85 method=Health result=2 took=32.967µs +ts=2025-08-04T11:54:39Z caller=logging.go:85 method=Health result=2 took=36.522µs +ts=2025-08-04T11:54:42Z caller=logging.go:85 method=Health result=2 took=31.748µs +ts=2025-08-04T11:54:42Z caller=logging.go:85 method=Health result=2 took=25.799µs +ts=2025-08-04T11:54:45Z caller=logging.go:85 method=Health result=2 took=30.523µs +ts=2025-08-04T11:54:45Z caller=logging.go:85 method=Health result=2 took=25.025µs +ts=2025-08-04T11:54:48Z caller=logging.go:85 method=Health result=2 took=31.886µs +ts=2025-08-04T11:54:48Z caller=logging.go:85 method=Health result=2 took=26.226µs +ts=2025-08-04T11:54:51Z caller=logging.go:85 method=Health result=2 took=47.258µs +ts=2025-08-04T11:54:51Z caller=logging.go:85 method=Health result=2 took=25.8µs +ts=2025-08-04T11:54:54Z caller=logging.go:85 method=Health result=2 took=31.409µs +ts=2025-08-04T11:54:54Z caller=logging.go:85 method=Health result=2 took=26.462µs +ts=2025-08-04T11:54:57Z caller=logging.go:85 method=Health result=2 took=24.978µs +ts=2025-08-04T11:54:57Z caller=logging.go:85 method=Health result=2 took=42.328µs +ts=2025-08-04T11:55:00Z caller=logging.go:85 method=Health result=2 took=41.958µs +ts=2025-08-04T11:55:00Z caller=logging.go:85 method=Health result=2 took=41.957µs +ts=2025-08-04T11:55:03Z caller=logging.go:85 method=Health result=2 took=32.469µs +ts=2025-08-04T11:55:03Z caller=logging.go:85 method=Health result=2 took=42.762µs +ts=2025-08-04T11:55:06Z caller=logging.go:85 method=Health result=2 took=33.22µs +ts=2025-08-04T11:55:06Z caller=logging.go:85 method=Health result=2 took=44.968µs +ts=2025-08-04T11:55:09Z caller=logging.go:85 method=Health result=2 took=42.192µs +ts=2025-08-04T11:55:09Z caller=logging.go:85 method=Health result=2 took=35.777µs +ts=2025-08-04T11:55:12Z caller=logging.go:85 method=Health result=2 took=34.444µs +ts=2025-08-04T11:55:12Z caller=logging.go:85 method=Health result=2 took=243.882µs +ts=2025-08-04T11:55:15Z caller=logging.go:85 method=Health result=2 took=42.494µs +ts=2025-08-04T11:55:15Z caller=logging.go:85 method=Health result=2 took=40.023µs +ts=2025-08-04T11:55:18Z caller=logging.go:85 method=Health result=2 took=31.603µs +ts=2025-08-04T11:55:18Z caller=logging.go:85 method=Health result=2 took=26.074µs +ts=2025-08-04T11:55:21Z caller=logging.go:85 method=Health result=2 took=30.904µs +ts=2025-08-04T11:55:21Z caller=logging.go:85 method=Health result=2 took=25.135µs +ts=2025-08-04T11:55:24Z caller=logging.go:85 method=Health result=2 took=32.289µs +ts=2025-08-04T11:55:24Z caller=logging.go:85 method=Health result=2 took=26.949µs +ts=2025-08-04T11:55:27Z caller=logging.go:85 method=Health result=2 took=31.79µs +ts=2025-08-04T11:55:27Z caller=logging.go:85 method=Health result=2 took=26.006µs +ts=2025-08-04T11:55:30Z caller=logging.go:85 method=Health result=2 took=26.308µs +ts=2025-08-04T11:55:30Z caller=logging.go:85 method=Health result=2 took=33.992µs +ts=2025-08-04T11:55:33Z caller=logging.go:85 method=Health result=2 took=32.677µs +ts=2025-08-04T11:55:33Z caller=logging.go:85 method=Health result=2 took=21.376µs +ts=2025-08-04T11:55:36Z caller=logging.go:85 method=Health result=2 took=32.182µs +ts=2025-08-04T11:55:36Z caller=logging.go:85 method=Health result=2 took=25.46µs +ts=2025-08-04T11:55:39Z caller=logging.go:85 method=Health result=2 took=49.718µs +ts=2025-08-04T11:55:39Z caller=logging.go:85 method=Health result=2 took=27.759µs +ts=2025-08-04T11:55:42Z caller=logging.go:85 method=Health result=2 took=25.788µs +ts=2025-08-04T11:55:42Z caller=logging.go:85 method=Health result=2 took=31.681µs +ts=2025-08-04T11:55:45Z caller=logging.go:85 method=Health result=2 took=45.955µs +ts=2025-08-04T11:55:45Z caller=logging.go:85 method=Health result=2 took=25.909µs +ts=2025-08-04T11:55:48Z caller=logging.go:85 method=Health result=2 took=30.045µs +ts=2025-08-04T11:55:48Z caller=logging.go:85 method=Health result=2 took=40.156µs +ts=2025-08-04T11:55:51Z caller=logging.go:85 method=Health result=2 took=30.146µs +ts=2025-08-04T11:55:51Z caller=logging.go:85 method=Health result=2 took=176.961µs +ts=2025-08-04T11:55:54Z caller=logging.go:85 method=Health result=2 took=43.197µs +ts=2025-08-04T11:55:54Z caller=logging.go:85 method=Health result=2 took=43.106µs +ts=2025-08-04T11:55:57Z caller=logging.go:85 method=Health result=2 took=32.634µs +ts=2025-08-04T11:55:57Z caller=logging.go:85 method=Health result=2 took=18.941µs +ts=2025-08-04T11:56:00Z caller=logging.go:85 method=Health result=2 took=41.393µs +ts=2025-08-04T11:56:00Z caller=logging.go:85 method=Health result=2 took=25.616µs +ts=2025-08-04T11:56:03Z caller=logging.go:85 method=Health result=2 took=32.404µs +ts=2025-08-04T11:56:03Z caller=logging.go:85 method=Health result=2 took=25.628µs +ts=2025-08-04T11:56:06Z caller=logging.go:85 method=Health result=2 took=32.057µs +ts=2025-08-04T11:56:06Z caller=logging.go:85 method=Health result=2 took=25.276µs +ts=2025-08-04T11:56:09Z caller=logging.go:85 method=Health result=2 took=32.397µs +ts=2025-08-04T11:56:09Z caller=logging.go:85 method=Health result=2 took=25.526µs +ts=2025-08-04T11:56:12Z caller=logging.go:85 method=Health result=2 took=35.732µs +ts=2025-08-04T11:56:12Z caller=logging.go:85 method=Health result=2 took=26.352µs +ts=2025-08-04T11:56:15Z caller=logging.go:85 method=Health result=2 took=28.228µs +ts=2025-08-04T11:56:15Z caller=logging.go:85 method=Health result=2 took=49.341µs +ts=2025-08-04T11:56:18Z caller=logging.go:85 method=Health result=2 took=33.216µs +ts=2025-08-04T11:56:18Z caller=logging.go:85 method=Health result=2 took=23.783µs +ts=2025-08-04T11:56:21Z caller=logging.go:85 method=Health result=2 took=40.457µs +ts=2025-08-04T11:56:21Z caller=logging.go:85 method=Health result=2 took=38.512µs +ts=2025-08-04T11:56:24Z caller=logging.go:85 method=Health result=2 took=45.754µs +ts=2025-08-04T11:56:24Z caller=logging.go:85 method=Health result=2 took=25.862µs +ts=2025-08-04T11:56:27Z caller=logging.go:85 method=Health result=2 took=31.594µs +ts=2025-08-04T11:56:27Z caller=logging.go:85 method=Health result=2 took=25.651µs +ts=2025-08-04T11:56:30Z caller=logging.go:85 method=Health result=2 took=32.18µs +ts=2025-08-04T11:56:30Z caller=logging.go:85 method=Health result=2 took=24.97µs +ts=2025-08-04T11:56:33Z caller=logging.go:85 method=Health result=2 took=41.02µs +ts=2025-08-04T11:56:33Z caller=logging.go:85 method=Health result=2 took=26.748µs +ts=2025-08-04T11:56:36Z caller=logging.go:85 method=Health result=2 took=31.874µs +ts=2025-08-04T11:56:36Z caller=logging.go:85 method=Health result=2 took=124.34µs +ts=2025-08-04T11:56:39Z caller=logging.go:85 method=Health result=2 took=32.646µs +ts=2025-08-04T11:56:39Z caller=logging.go:85 method=Health result=2 took=25.58µs +ts=2025-08-04T11:56:42Z caller=logging.go:85 method=Health result=2 took=35.233µs +ts=2025-08-04T11:56:42Z caller=logging.go:85 method=Health result=2 took=23.331µs +ts=2025-08-04T11:56:45Z caller=logging.go:85 method=Health result=2 took=25.838µs +ts=2025-08-04T11:56:45Z caller=logging.go:85 method=Health result=2 took=31.527µs +ts=2025-08-04T11:56:48Z caller=logging.go:85 method=Health result=2 took=36.858µs +ts=2025-08-04T11:56:48Z caller=logging.go:85 method=Health result=2 took=25.657µs +ts=2025-08-04T11:56:51Z caller=logging.go:85 method=Health result=2 took=31.896µs +ts=2025-08-04T11:56:51Z caller=logging.go:85 method=Health result=2 took=26.888µs +ts=2025-08-04T11:56:54Z caller=logging.go:85 method=Health result=2 took=33.495µs +ts=2025-08-04T11:56:54Z caller=logging.go:85 method=Health result=2 took=42.176µs +ts=2025-08-04T11:56:57Z caller=logging.go:85 method=Health result=2 took=30.463µs +ts=2025-08-04T11:56:57Z caller=logging.go:85 method=Health result=2 took=25.269µs +ts=2025-08-04T11:57:00Z caller=logging.go:85 method=Health result=2 took=29.842µs +ts=2025-08-04T11:57:00Z caller=logging.go:85 method=Health result=2 took=22.487µs +ts=2025-08-04T11:57:03Z caller=logging.go:85 method=Health result=2 took=32.247µs +ts=2025-08-04T11:57:03Z caller=logging.go:85 method=Health result=2 took=24.86µs +ts=2025-08-04T11:57:06Z caller=logging.go:85 method=Health result=2 took=31.361µs +ts=2025-08-04T11:57:06Z caller=logging.go:85 method=Health result=2 took=24.677µs +ts=2025-08-04T11:57:09Z caller=logging.go:85 method=Health result=2 took=33.224µs +ts=2025-08-04T11:57:09Z caller=logging.go:85 method=Health result=2 took=24.481µs +ts=2025-08-04T11:57:12Z caller=logging.go:85 method=Health result=2 took=41.504µs +ts=2025-08-04T11:57:12Z caller=logging.go:85 method=Health result=2 took=41.505µs +ts=2025-08-04T11:57:15Z caller=logging.go:85 method=Health result=2 took=31.039µs +ts=2025-08-04T11:57:15Z caller=logging.go:85 method=Health result=2 took=25.515µs +ts=2025-08-04T11:57:18Z caller=logging.go:85 method=Health result=2 took=42.565µs +ts=2025-08-04T11:57:18Z caller=logging.go:85 method=Health result=2 took=42.557µs +ts=2025-08-04T11:57:21Z caller=logging.go:85 method=Health result=2 took=33.475µs +ts=2025-08-04T11:57:21Z caller=logging.go:85 method=Health result=2 took=200.45µs +ts=2025-08-04T11:57:24Z caller=logging.go:85 method=Health result=2 took=32.833µs +ts=2025-08-04T11:57:24Z caller=logging.go:85 method=Health result=2 took=40.864µs +ts=2025-08-04T11:57:27Z caller=logging.go:85 method=Health result=2 took=40.076µs +ts=2025-08-04T11:57:27Z caller=logging.go:85 method=Health result=2 took=57.513µs +ts=2025-08-04T11:57:30Z caller=logging.go:85 method=Health result=2 took=31.209µs +ts=2025-08-04T11:57:30Z caller=logging.go:85 method=Health result=2 took=24.865µs +ts=2025-08-04T11:57:33Z caller=logging.go:85 method=Health result=2 took=29.115µs +ts=2025-08-04T11:57:33Z caller=logging.go:85 method=Health result=2 took=43.302µs +ts=2025-08-04T11:57:36Z caller=logging.go:85 method=Health result=2 took=42.073µs +ts=2025-08-04T11:57:36Z caller=logging.go:85 method=Health result=2 took=31.969µs +ts=2025-08-04T11:57:39Z caller=logging.go:85 method=Health result=2 took=33.579µs +ts=2025-08-04T11:57:39Z caller=logging.go:85 method=Health result=2 took=24.897µs +ts=2025-08-04T11:57:42Z caller=logging.go:85 method=Health result=2 took=32.095µs +ts=2025-08-04T11:57:42Z caller=logging.go:85 method=Health result=2 took=37.933µs +ts=2025-08-04T11:57:45Z caller=logging.go:85 method=Health result=2 took=29.78µs +ts=2025-08-04T11:57:45Z caller=logging.go:85 method=Health result=2 took=27.81µs +ts=2025-08-04T11:57:48Z caller=logging.go:85 method=Health result=2 took=35.574µs +ts=2025-08-04T11:57:48Z caller=logging.go:85 method=Health result=2 took=24.782µs +ts=2025-08-04T11:57:51Z caller=logging.go:85 method=Health result=2 took=40.89µs +ts=2025-08-04T11:57:51Z caller=logging.go:85 method=Health result=2 took=31.999µs +ts=2025-08-04T11:57:54Z caller=logging.go:85 method=Health result=2 took=31.814µs +ts=2025-08-04T11:57:54Z caller=logging.go:85 method=Health result=2 took=26.367µs +ts=2025-08-04T11:57:57Z caller=logging.go:85 method=Health result=2 took=41.937µs +ts=2025-08-04T11:57:57Z caller=logging.go:85 method=Health result=2 took=25.147µs +ts=2025-08-04T11:58:00Z caller=logging.go:85 method=Health result=2 took=32.253µs +ts=2025-08-04T11:58:00Z caller=logging.go:85 method=Health result=2 took=42.061µs +ts=2025-08-04T11:58:03Z caller=logging.go:85 method=Health result=2 took=44.267µs +ts=2025-08-04T11:58:03Z caller=logging.go:85 method=Health result=2 took=36.555µs +ts=2025-08-04T11:58:06Z caller=logging.go:85 method=Health result=2 took=41.912µs +ts=2025-08-04T11:58:06Z caller=logging.go:85 method=Health result=2 took=98.133µs +ts=2025-08-04T11:58:09Z caller=logging.go:85 method=Health result=2 took=44.462µs +ts=2025-08-04T11:58:09Z caller=logging.go:85 method=Health result=2 took=40.154µs +ts=2025-08-04T11:58:12Z caller=logging.go:85 method=Health result=2 took=31.576µs +ts=2025-08-04T11:58:12Z caller=logging.go:85 method=Health result=2 took=25.529µs +ts=2025-08-04T11:58:15Z caller=logging.go:85 method=Health result=2 took=30.758µs +ts=2025-08-04T11:58:15Z caller=logging.go:85 method=Health result=2 took=27.166µs +ts=2025-08-04T11:58:18Z caller=logging.go:85 method=Health result=2 took=33.631µs +ts=2025-08-04T11:58:18Z caller=logging.go:85 method=Health result=2 took=25.644µs +ts=2025-08-04T11:58:21Z caller=logging.go:85 method=Health result=2 took=33.371µs +ts=2025-08-04T11:58:21Z caller=logging.go:85 method=Health result=2 took=29.962µs +ts=2025-08-04T11:58:24Z caller=logging.go:85 method=Health result=2 took=41.865µs +ts=2025-08-04T11:58:24Z caller=logging.go:85 method=Health result=2 took=41.87µs +ts=2025-08-04T11:58:27Z caller=logging.go:85 method=Health result=2 took=30.771µs +ts=2025-08-04T11:58:27Z caller=logging.go:85 method=Health result=2 took=27.613µs +ts=2025-08-04T11:58:30Z caller=logging.go:85 method=Health result=2 took=31.142µs +ts=2025-08-04T11:58:30Z caller=logging.go:85 method=Health result=2 took=27.287µs +ts=2025-08-04T11:58:33Z caller=logging.go:85 method=Health result=2 took=52.916µs +ts=2025-08-04T11:58:33Z caller=logging.go:85 method=Health result=2 took=43.373µs +ts=2025-08-04T11:58:36Z caller=logging.go:85 method=Health result=2 took=40.705µs +ts=2025-08-04T11:58:36Z caller=logging.go:85 method=Health result=2 took=34.934µs +ts=2025-08-04T11:58:39Z caller=logging.go:85 method=Health result=2 took=44.016µs +ts=2025-08-04T11:58:39Z caller=logging.go:85 method=Health result=2 took=20.66µs +ts=2025-08-04T11:58:42Z caller=logging.go:85 method=Health result=2 took=41.979µs +ts=2025-08-04T11:58:42Z caller=logging.go:85 method=Health result=2 took=41.981µs +ts=2025-08-04T11:58:45Z caller=logging.go:85 method=Health result=2 took=43.797µs +ts=2025-08-04T11:58:45Z caller=logging.go:85 method=Health result=2 took=25.666µs +ts=2025-08-04T11:58:48Z caller=logging.go:85 method=Health result=2 took=33.989µs +ts=2025-08-04T11:58:48Z caller=logging.go:85 method=Health result=2 took=26.077µs +ts=2025-08-04T11:58:51Z caller=logging.go:85 method=Health result=2 took=30.947µs +ts=2025-08-04T11:58:51Z caller=logging.go:85 method=Health result=2 took=25.199µs +ts=2025-08-04T11:58:54Z caller=logging.go:85 method=Health result=2 took=31.973µs +ts=2025-08-04T11:58:54Z caller=logging.go:85 method=Health result=2 took=24.739µs +ts=2025-08-04T11:58:57Z caller=logging.go:85 method=Health result=2 took=33.423µs +ts=2025-08-04T11:58:57Z caller=logging.go:85 method=Health result=2 took=38.359µs +ts=2025-08-04T11:59:00Z caller=logging.go:85 method=Health result=2 took=32.507µs +ts=2025-08-04T11:59:00Z caller=logging.go:85 method=Health result=2 took=25.639µs +ts=2025-08-04T11:59:03Z caller=logging.go:85 method=Health result=2 took=31.532µs +ts=2025-08-04T11:59:03Z caller=logging.go:85 method=Health result=2 took=25.425µs +ts=2025-08-04T11:59:06Z caller=logging.go:85 method=Health result=2 took=32.036µs +ts=2025-08-04T11:59:06Z caller=logging.go:85 method=Health result=2 took=24.271µs +ts=2025-08-04T11:59:09Z caller=logging.go:85 method=Health result=2 took=30.4µs +ts=2025-08-04T11:59:09Z caller=logging.go:85 method=Health result=2 took=43.273µs +ts=2025-08-04T11:59:12Z caller=logging.go:85 method=Health result=2 took=31.738µs +ts=2025-08-04T11:59:12Z caller=logging.go:85 method=Health result=2 took=42.352µs +ts=2025-08-04T11:59:15Z caller=logging.go:85 method=Health result=2 took=33.866µs +ts=2025-08-04T11:59:15Z caller=logging.go:85 method=Health result=2 took=28.006µs +ts=2025-08-04T11:59:18Z caller=logging.go:85 method=Health result=2 took=32.898µs +ts=2025-08-04T11:59:18Z caller=logging.go:85 method=Health result=2 took=25.936µs +ts=2025-08-04T11:59:21Z caller=logging.go:85 method=Health result=2 took=29.469µs +ts=2025-08-04T11:59:21Z caller=logging.go:85 method=Health result=2 took=27.661µs +ts=2025-08-04T11:59:24Z caller=logging.go:85 method=Health result=2 took=48.39µs +ts=2025-08-04T11:59:24Z caller=logging.go:85 method=Health result=2 took=56.683µs +ts=2025-08-04T11:59:27Z caller=logging.go:85 method=Health result=2 took=42.486µs +ts=2025-08-04T11:59:27Z caller=logging.go:85 method=Health result=2 took=36.111µs +ts=2025-08-04T11:59:30Z caller=logging.go:85 method=Health result=2 took=31.643µs +ts=2025-08-04T11:59:30Z caller=logging.go:85 method=Health result=2 took=41.226µs +ts=2025-08-04T11:59:33Z caller=logging.go:85 method=Health result=2 took=31.603µs +ts=2025-08-04T11:59:33Z caller=logging.go:85 method=Health result=2 took=26.574µs +ts=2025-08-04T11:59:36Z caller=logging.go:85 method=Health result=2 took=32.151µs +ts=2025-08-04T11:59:36Z caller=logging.go:85 method=Health result=2 took=24.817µs +ts=2025-08-04T11:59:39Z caller=logging.go:85 method=Health result=2 took=38.844µs +ts=2025-08-04T11:59:39Z caller=logging.go:85 method=Health result=2 took=25.718µs +ts=2025-08-04T11:59:42Z caller=logging.go:85 method=Health result=2 took=32.946µs +ts=2025-08-04T11:59:42Z caller=logging.go:85 method=Health result=2 took=25.145µs +ts=2025-08-04T11:59:45Z caller=logging.go:85 method=Health result=2 took=33.426µs +ts=2025-08-04T11:59:45Z caller=logging.go:85 method=Health result=2 took=25.212µs +ts=2025-08-04T11:59:48Z caller=logging.go:85 method=Health result=2 took=26.177µs +ts=2025-08-04T11:59:48Z caller=logging.go:85 method=Health result=2 took=32.98µs +ts=2025-08-04T11:59:51Z caller=logging.go:85 method=Health result=2 took=32µs +ts=2025-08-04T11:59:51Z caller=logging.go:85 method=Health result=2 took=211.731µs +ts=2025-08-04T11:59:54Z caller=logging.go:85 method=Health result=2 took=34.436µs +ts=2025-08-04T11:59:54Z caller=logging.go:85 method=Health result=2 took=25.762µs +ts=2025-08-04T11:59:57Z caller=logging.go:85 method=Health result=2 took=32.144µs +ts=2025-08-04T11:59:57Z caller=logging.go:85 method=Health result=2 took=28.581µs +ts=2025-08-04T12:00:00Z caller=logging.go:85 method=Health result=2 took=54.113µs +ts=2025-08-04T12:00:00Z caller=logging.go:85 method=Health result=2 took=44.807µs +ts=2025-08-04T12:00:03Z caller=logging.go:85 method=Health result=2 took=44.926µs +ts=2025-08-04T12:00:03Z caller=logging.go:85 method=Health result=2 took=40.527µs +ts=2025-08-04T12:00:06Z caller=logging.go:85 method=Health result=2 took=32.193µs +ts=2025-08-04T12:00:06Z caller=logging.go:85 method=Health result=2 took=38.299µs +ts=2025-08-04T12:00:09Z caller=logging.go:85 method=Health result=2 took=38.293µs +ts=2025-08-04T12:00:09Z caller=logging.go:85 method=Health result=2 took=26.426µs +ts=2025-08-04T12:00:12Z caller=logging.go:85 method=Health result=2 took=42.476µs +ts=2025-08-04T12:00:12Z caller=logging.go:85 method=Health result=2 took=42.475µs +ts=2025-08-04T12:00:15Z caller=logging.go:85 method=Health result=2 took=27.344µs +ts=2025-08-04T12:00:15Z caller=logging.go:85 method=Health result=2 took=35.264µs +ts=2025-08-04T12:00:18Z caller=logging.go:85 method=Health result=2 took=48.15µs +ts=2025-08-04T12:00:18Z caller=logging.go:85 method=Health result=2 took=35.144µs +ts=2025-08-04T12:00:21Z caller=logging.go:85 method=Health result=2 took=32.952µs +ts=2025-08-04T12:00:21Z caller=logging.go:85 method=Health result=2 took=28.503µs +ts=2025-08-04T12:00:24Z caller=logging.go:85 method=Health result=2 took=31.802µs +ts=2025-08-04T12:00:24Z caller=logging.go:85 method=Health result=2 took=30.137µs +ts=2025-08-04T12:00:27Z caller=logging.go:85 method=Health result=2 took=42.407µs +ts=2025-08-04T12:00:27Z caller=logging.go:85 method=Health result=2 took=42.41µs +ts=2025-08-04T12:00:30Z caller=logging.go:85 method=Health result=2 took=42.443µs +ts=2025-08-04T12:00:30Z caller=logging.go:85 method=Health result=2 took=24.597µs +ts=2025-08-04T12:00:33Z caller=logging.go:85 method=Health result=2 took=32.52µs +ts=2025-08-04T12:00:33Z caller=logging.go:85 method=Health result=2 took=44.128µs +ts=2025-08-04T12:00:36Z caller=logging.go:85 method=Health result=2 took=32.207µs +ts=2025-08-04T12:00:36Z caller=logging.go:85 method=Health result=2 took=25.511µs +ts=2025-08-04T12:00:39Z caller=logging.go:85 method=Health result=2 took=35.864µs +ts=2025-08-04T12:00:39Z caller=logging.go:85 method=Health result=2 took=32.235µs +ts=2025-08-04T12:00:42Z caller=logging.go:85 method=Health result=2 took=35.512µs +ts=2025-08-04T12:00:42Z caller=logging.go:85 method=Health result=2 took=25.486µs +ts=2025-08-04T12:00:45Z caller=logging.go:85 method=Health result=2 took=32.163µs +ts=2025-08-04T12:00:45Z caller=logging.go:85 method=Health result=2 took=21.897µs +ts=2025-08-04T12:00:48Z caller=logging.go:85 method=Health result=2 took=30.487µs +ts=2025-08-04T12:00:48Z caller=logging.go:85 method=Health result=2 took=26.425µs +ts=2025-08-04T12:00:51Z caller=logging.go:85 method=Health result=2 took=32.778µs +ts=2025-08-04T12:00:51Z caller=logging.go:85 method=Health result=2 took=52.74µs +ts=2025-08-04T12:00:54Z caller=logging.go:85 method=Health result=2 took=33.728µs +ts=2025-08-04T12:00:54Z caller=logging.go:85 method=Health result=2 took=25.808µs +ts=2025-08-04T12:00:57Z caller=logging.go:85 method=Health result=2 took=32.716µs +ts=2025-08-04T12:00:57Z caller=logging.go:85 method=Health result=2 took=25.287µs +ts=2025-08-04T12:01:00Z caller=logging.go:85 method=Health result=2 took=33.063µs +ts=2025-08-04T12:01:00Z caller=logging.go:85 method=Health result=2 took=20.207µs +ts=2025-08-04T12:01:03Z caller=logging.go:85 method=Health result=2 took=19.042µs +ts=2025-08-04T12:01:03Z caller=logging.go:85 method=Health result=2 took=43.805µs +ts=2025-08-04T12:01:06Z caller=logging.go:85 method=Health result=2 took=32.06µs +ts=2025-08-04T12:01:06Z caller=logging.go:85 method=Health result=2 took=25.375µs +ts=2025-08-04T12:01:09Z caller=logging.go:85 method=Health result=2 took=31.651µs +ts=2025-08-04T12:01:09Z caller=logging.go:85 method=Health result=2 took=26.403µs +ts=2025-08-04T12:01:12Z caller=logging.go:85 method=Health result=2 took=35.345µs +ts=2025-08-04T12:01:12Z caller=logging.go:85 method=Health result=2 took=37.199µs +ts=2025-08-04T12:01:15Z caller=logging.go:85 method=Health result=2 took=45.994µs +ts=2025-08-04T12:01:15Z caller=logging.go:85 method=Health result=2 took=47.277µs +ts=2025-08-04T12:01:18Z caller=logging.go:85 method=Health result=2 took=34.186µs +ts=2025-08-04T12:01:18Z caller=logging.go:85 method=Health result=2 took=157.132µs +ts=2025-08-04T12:01:21Z caller=logging.go:85 method=Health result=2 took=31.678µs +ts=2025-08-04T12:01:21Z caller=logging.go:85 method=Health result=2 took=25.595µs +ts=2025-08-04T12:01:24Z caller=logging.go:85 method=Health result=2 took=31.85µs +ts=2025-08-04T12:01:24Z caller=logging.go:85 method=Health result=2 took=25.572µs +ts=2025-08-04T12:01:27Z caller=logging.go:85 method=Health result=2 took=41.171µs +ts=2025-08-04T12:01:27Z caller=logging.go:85 method=Health result=2 took=32.24µs +ts=2025-08-04T12:01:30Z caller=logging.go:85 method=Health result=2 took=42.891µs +ts=2025-08-04T12:01:30Z caller=logging.go:85 method=Health result=2 took=25.381µs +ts=2025-08-04T12:01:33Z caller=logging.go:85 method=Health result=2 took=42.962µs +ts=2025-08-04T12:01:33Z caller=logging.go:85 method=Health result=2 took=35.834µs +ts=2025-08-04T12:01:36Z caller=logging.go:85 method=Health result=2 took=33.101µs +ts=2025-08-04T12:01:36Z caller=logging.go:85 method=Health result=2 took=16.313µs +ts=2025-08-04T12:01:39Z caller=logging.go:85 method=Health result=2 took=32.103µs +ts=2025-08-04T12:01:39Z caller=logging.go:85 method=Health result=2 took=24.696µs +ts=2025-08-04T12:01:42Z caller=logging.go:85 method=Health result=2 took=37.469µs +ts=2025-08-04T12:01:42Z caller=logging.go:85 method=Health result=2 took=41.082µs +ts=2025-08-04T12:01:45Z caller=logging.go:85 method=Health result=2 took=31.114µs +ts=2025-08-04T12:01:45Z caller=logging.go:85 method=Health result=2 took=24.592µs +ts=2025-08-04T12:01:48Z caller=logging.go:85 method=Health result=2 took=31.568µs +ts=2025-08-04T12:01:48Z caller=logging.go:85 method=Health result=2 took=94.499µs +ts=2025-08-04T12:01:51Z caller=logging.go:85 method=Health result=2 took=33.562µs +ts=2025-08-04T12:01:51Z caller=logging.go:85 method=Health result=2 took=26.211µs +ts=2025-08-04T12:01:54Z caller=logging.go:85 method=Health result=2 took=31.727µs +ts=2025-08-04T12:01:54Z caller=logging.go:85 method=Health result=2 took=25.988µs +ts=2025-08-04T12:01:57Z caller=logging.go:85 method=Health result=2 took=46.583µs +ts=2025-08-04T12:01:57Z caller=logging.go:85 method=Health result=2 took=29.748µs +ts=2025-08-04T12:02:00Z caller=logging.go:85 method=Health result=2 took=31.584µs +ts=2025-08-04T12:02:00Z caller=logging.go:85 method=Health result=2 took=26.965µs +ts=2025-08-04T12:02:03Z caller=logging.go:85 method=Health result=2 took=33.07µs +ts=2025-08-04T12:02:03Z caller=logging.go:85 method=Health result=2 took=28.444µs +ts=2025-08-04T12:02:06Z caller=logging.go:85 method=Health result=2 took=41.558µs +ts=2025-08-04T12:02:06Z caller=logging.go:85 method=Health result=2 took=41.068µs +ts=2025-08-04T12:02:09Z caller=logging.go:85 method=Health result=2 took=44.067µs +ts=2025-08-04T12:02:09Z caller=logging.go:85 method=Health result=2 took=25.42µs +ts=2025-08-04T12:02:12Z caller=logging.go:85 method=Health result=2 took=31.142µs +ts=2025-08-04T12:02:12Z caller=logging.go:85 method=Health result=2 took=25.004µs +ts=2025-08-04T12:02:15Z caller=logging.go:85 method=Health result=2 took=44.926µs +ts=2025-08-04T12:02:15Z caller=logging.go:85 method=Health result=2 took=27.154µs +ts=2025-08-04T12:02:18Z caller=logging.go:85 method=Health result=2 took=31.879µs +ts=2025-08-04T12:02:18Z caller=logging.go:85 method=Health result=2 took=25.339µs +ts=2025-08-04T12:02:21Z caller=logging.go:85 method=Health result=2 took=32.957µs +ts=2025-08-04T12:02:21Z caller=logging.go:85 method=Health result=2 took=25.004µs +ts=2025-08-04T12:02:24Z caller=logging.go:85 method=Health result=2 took=30.282µs +ts=2025-08-04T12:02:24Z caller=logging.go:85 method=Health result=2 took=39.234µs +ts=2025-08-04T12:02:27Z caller=logging.go:85 method=Health result=2 took=32.795µs +ts=2025-08-04T12:02:27Z caller=logging.go:85 method=Health result=2 took=36.444µs +ts=2025-08-04T12:02:30Z caller=logging.go:85 method=Health result=2 took=33.921µs +ts=2025-08-04T12:02:30Z caller=logging.go:85 method=Health result=2 took=48.029µs +ts=2025-08-04T12:02:33Z caller=logging.go:85 method=Health result=2 took=31.618µs +ts=2025-08-04T12:02:33Z caller=logging.go:85 method=Health result=2 took=26.05µs +ts=2025-08-04T12:02:36Z caller=logging.go:85 method=Health result=2 took=32.709µs +ts=2025-08-04T12:02:36Z caller=logging.go:85 method=Health result=2 took=25.779µs +ts=2025-08-04T12:02:39Z caller=logging.go:85 method=Health result=2 took=41.236µs +ts=2025-08-04T12:02:39Z caller=logging.go:85 method=Health result=2 took=41.238µs +ts=2025-08-04T12:02:42Z caller=logging.go:85 method=Health result=2 took=41.209µs +ts=2025-08-04T12:02:42Z caller=logging.go:85 method=Health result=2 took=33.173µs +ts=2025-08-04T12:02:45Z caller=logging.go:85 method=Health result=2 took=33.531µs +ts=2025-08-04T12:02:45Z caller=logging.go:85 method=Health result=2 took=26.782µs +ts=2025-08-04T12:02:48Z caller=logging.go:85 method=Health result=2 took=32.977µs +ts=2025-08-04T12:02:48Z caller=logging.go:85 method=Health result=2 took=26.925µs +ts=2025-08-04T12:02:51Z caller=logging.go:85 method=Health result=2 took=32.065µs +ts=2025-08-04T12:02:51Z caller=logging.go:85 method=Health result=2 took=25.136µs +ts=2025-08-04T12:02:54Z caller=logging.go:85 method=Health result=2 took=31.953µs +ts=2025-08-04T12:02:54Z caller=logging.go:85 method=Health result=2 took=41.745µs +ts=2025-08-04T12:02:57Z caller=logging.go:85 method=Health result=2 took=46.439µs +ts=2025-08-04T12:02:57Z caller=logging.go:85 method=Health result=2 took=26.705µs +ts=2025-08-04T12:03:00Z caller=logging.go:85 method=Health result=2 took=32.559µs +ts=2025-08-04T12:03:00Z caller=logging.go:85 method=Health result=2 took=24.911µs +ts=2025-08-04T12:03:03Z caller=logging.go:85 method=Health result=2 took=31.394µs +ts=2025-08-04T12:03:03Z caller=logging.go:85 method=Health result=2 took=24.497µs +ts=2025-08-04T12:03:06Z caller=logging.go:85 method=Health result=2 took=32.378µs +ts=2025-08-04T12:03:06Z caller=logging.go:85 method=Health result=2 took=25.21µs +ts=2025-08-04T12:03:09Z caller=logging.go:85 method=Health result=2 took=24.388µs +ts=2025-08-04T12:03:09Z caller=logging.go:85 method=Health result=2 took=33.173µs +ts=2025-08-04T12:03:12Z caller=logging.go:85 method=Health result=2 took=32.649µs +ts=2025-08-04T12:03:12Z caller=logging.go:85 method=Health result=2 took=20.687µs +ts=2025-08-04T12:03:15Z caller=logging.go:85 method=Health result=2 took=33.161µs +ts=2025-08-04T12:03:15Z caller=logging.go:85 method=Health result=2 took=25.213µs +ts=2025-08-04T12:03:18Z caller=logging.go:85 method=Health result=2 took=31.907µs +ts=2025-08-04T12:03:18Z caller=logging.go:85 method=Health result=2 took=26.776µs +ts=2025-08-04T12:03:21Z caller=logging.go:85 method=Health result=2 took=31.204µs +ts=2025-08-04T12:03:21Z caller=logging.go:85 method=Health result=2 took=26.603µs +ts=2025-08-04T12:03:24Z caller=logging.go:85 method=Health result=2 took=32.219µs +ts=2025-08-04T12:03:24Z caller=logging.go:85 method=Health result=2 took=25.499µs +ts=2025-08-04T12:03:27Z caller=logging.go:85 method=Health result=2 took=26.711µs +ts=2025-08-04T12:03:27Z caller=logging.go:85 method=Health result=2 took=42.661µs +ts=2025-08-04T12:03:30Z caller=logging.go:85 method=Health result=2 took=30.602µs +ts=2025-08-04T12:03:30Z caller=logging.go:85 method=Health result=2 took=24.939µs +ts=2025-08-04T12:03:33Z caller=logging.go:85 method=Health result=2 took=32.315µs +ts=2025-08-04T12:03:33Z caller=logging.go:85 method=Health result=2 took=25.524µs +ts=2025-08-04T12:03:36Z caller=logging.go:85 method=Health result=2 took=45.393µs +ts=2025-08-04T12:03:36Z caller=logging.go:85 method=Health result=2 took=45.397µs +ts=2025-08-04T12:03:39Z caller=logging.go:85 method=Health result=2 took=33.767µs +ts=2025-08-04T12:03:39Z caller=logging.go:85 method=Health result=2 took=27.65µs +ts=2025-08-04T12:03:42Z caller=logging.go:85 method=Health result=2 took=32.451µs +ts=2025-08-04T12:03:42Z caller=logging.go:85 method=Health result=2 took=25.543µs +ts=2025-08-04T12:03:45Z caller=logging.go:85 method=Health result=2 took=32.159µs +ts=2025-08-04T12:03:45Z caller=logging.go:85 method=Health result=2 took=25.694µs +ts=2025-08-04T12:03:48Z caller=logging.go:85 method=Health result=2 took=31.116µs +ts=2025-08-04T12:03:48Z caller=logging.go:85 method=Health result=2 took=27.334µs +ts=2025-08-04T12:03:51Z caller=logging.go:85 method=Health result=2 took=32.406µs +ts=2025-08-04T12:03:51Z caller=logging.go:85 method=Health result=2 took=25.061µs +ts=2025-08-04T12:03:54Z caller=logging.go:85 method=Health result=2 took=32.641µs +ts=2025-08-04T12:03:54Z caller=logging.go:85 method=Health result=2 took=29.3µs +ts=2025-08-04T12:03:57Z caller=logging.go:85 method=Health result=2 took=45.588µs +ts=2025-08-04T12:03:57Z caller=logging.go:85 method=Health result=2 took=19.762µs +ts=2025-08-04T12:04:00Z caller=logging.go:85 method=Health result=2 took=28.827µs +ts=2025-08-04T12:04:00Z caller=logging.go:85 method=Health result=2 took=49.885µs +ts=2025-08-04T12:04:03Z caller=logging.go:85 method=Health result=2 took=55.052µs +ts=2025-08-04T12:04:03Z caller=logging.go:85 method=Health result=2 took=20.213µs +ts=2025-08-04T12:04:06Z caller=logging.go:85 method=Health result=2 took=32.35µs +ts=2025-08-04T12:04:06Z caller=logging.go:85 method=Health result=2 took=25.883µs +ts=2025-08-04T12:04:09Z caller=logging.go:85 method=Health result=2 took=31.131µs +ts=2025-08-04T12:04:09Z caller=logging.go:85 method=Health result=2 took=24.546µs +ts=2025-08-04T12:04:12Z caller=logging.go:85 method=Health result=2 took=32.324µs +ts=2025-08-04T12:04:12Z caller=logging.go:85 method=Health result=2 took=39.668µs +ts=2025-08-04T12:04:15Z caller=logging.go:85 method=Health result=2 took=31.207µs +ts=2025-08-04T12:04:15Z caller=logging.go:85 method=Health result=2 took=25.084µs +ts=2025-08-04T12:04:18Z caller=logging.go:85 method=Health result=2 took=31.434µs +ts=2025-08-04T12:04:18Z caller=logging.go:85 method=Health result=2 took=25.45µs +ts=2025-08-04T12:04:21Z caller=logging.go:85 method=Health result=2 took=33.981µs +ts=2025-08-04T12:04:21Z caller=logging.go:85 method=Health result=2 took=26.224µs +ts=2025-08-04T12:04:24Z caller=logging.go:85 method=Health result=2 took=40.612µs +ts=2025-08-04T12:04:24Z caller=logging.go:85 method=Health result=2 took=36.277µs +ts=2025-08-04T12:04:27Z caller=logging.go:85 method=Health result=2 took=32.063µs +ts=2025-08-04T12:04:27Z caller=logging.go:85 method=Health result=2 took=25.944µs +ts=2025-08-04T12:04:30Z caller=logging.go:85 method=Health result=2 took=31.737µs +ts=2025-08-04T12:04:30Z caller=logging.go:85 method=Health result=2 took=199.283µs +ts=2025-08-04T12:04:33Z caller=logging.go:85 method=Health result=2 took=34.743µs +ts=2025-08-04T12:04:33Z caller=logging.go:85 method=Health result=2 took=39.471µs +ts=2025-08-04T12:04:36Z caller=logging.go:85 method=Health result=2 took=31.7µs +ts=2025-08-04T12:04:36Z caller=logging.go:85 method=Health result=2 took=25.392µs +ts=2025-08-04T12:04:39Z caller=logging.go:85 method=Health result=2 took=32.246µs +ts=2025-08-04T12:04:39Z caller=logging.go:85 method=Health result=2 took=25.562µs +ts=2025-08-04T12:04:42Z caller=logging.go:85 method=Health result=2 took=31.689µs +ts=2025-08-04T12:04:42Z caller=logging.go:85 method=Health result=2 took=40.323µs +ts=2025-08-04T12:04:45Z caller=logging.go:85 method=Health result=2 took=39.765µs +ts=2025-08-04T12:04:45Z caller=logging.go:85 method=Health result=2 took=44.929µs +ts=2025-08-04T12:04:48Z caller=logging.go:85 method=Health result=2 took=42.007µs +ts=2025-08-04T12:04:48Z caller=logging.go:85 method=Health result=2 took=33.32µs +ts=2025-08-04T12:04:51Z caller=logging.go:85 method=Health result=2 took=44.178µs +ts=2025-08-04T12:04:51Z caller=logging.go:85 method=Health result=2 took=44.139µs +ts=2025-08-04T12:04:54Z caller=logging.go:85 method=Health result=2 took=31.684µs +ts=2025-08-04T12:04:54Z caller=logging.go:85 method=Health result=2 took=26.628µs +ts=2025-08-04T12:04:57Z caller=logging.go:85 method=Health result=2 took=31.584µs +ts=2025-08-04T12:04:57Z caller=logging.go:85 method=Health result=2 took=26.204µs +ts=2025-08-04T12:05:00Z caller=logging.go:85 method=Health result=2 took=41.506µs +ts=2025-08-04T12:05:00Z caller=logging.go:85 method=Health result=2 took=56.624µs +ts=2025-08-04T12:05:03Z caller=logging.go:85 method=Health result=2 took=26.216µs +ts=2025-08-04T12:05:03Z caller=logging.go:85 method=Health result=2 took=44.164µs +ts=2025-08-04T12:05:06Z caller=logging.go:85 method=Health result=2 took=41.752µs +ts=2025-08-04T12:05:06Z caller=logging.go:85 method=Health result=2 took=50.254µs +ts=2025-08-04T12:05:09Z caller=logging.go:85 method=Health result=2 took=32.092µs +ts=2025-08-04T12:05:09Z caller=logging.go:85 method=Health result=2 took=16.461µs +ts=2025-08-04T12:05:12Z caller=logging.go:85 method=Health result=2 took=31.398µs +ts=2025-08-04T12:05:12Z caller=logging.go:85 method=Health result=2 took=30.061µs +ts=2025-08-04T12:05:15Z caller=logging.go:85 method=Health result=2 took=31.305µs +ts=2025-08-04T12:05:15Z caller=logging.go:85 method=Health result=2 took=25.14µs +ts=2025-08-04T12:05:18Z caller=logging.go:85 method=Health result=2 took=32.941µs +ts=2025-08-04T12:05:18Z caller=logging.go:85 method=Health result=2 took=25.329µs +ts=2025-08-04T12:05:21Z caller=logging.go:85 method=Health result=2 took=42.049µs +ts=2025-08-04T12:05:21Z caller=logging.go:85 method=Health result=2 took=37.755µs +ts=2025-08-04T12:05:24Z caller=logging.go:85 method=Health result=2 took=30.204µs +ts=2025-08-04T12:05:24Z caller=logging.go:85 method=Health result=2 took=41.733µs +ts=2025-08-04T12:05:27Z caller=logging.go:85 method=Health result=2 took=34.476µs +ts=2025-08-04T12:05:27Z caller=logging.go:85 method=Health result=2 took=25.22µs +ts=2025-08-04T12:05:30Z caller=logging.go:85 method=Health result=2 took=31.4µs +ts=2025-08-04T12:05:30Z caller=logging.go:85 method=Health result=2 took=27.211µs +ts=2025-08-04T12:05:33Z caller=logging.go:85 method=Health result=2 took=31.688µs +ts=2025-08-04T12:05:33Z caller=logging.go:85 method=Health result=2 took=26.363µs +ts=2025-08-04T12:05:36Z caller=logging.go:85 method=Health result=2 took=31.429µs +ts=2025-08-04T12:05:36Z caller=logging.go:85 method=Health result=2 took=38.303µs +ts=2025-08-04T12:05:39Z caller=logging.go:85 method=Health result=2 took=43.945µs +ts=2025-08-04T12:05:39Z caller=logging.go:85 method=Health result=2 took=26.396µs +ts=2025-08-04T12:05:42Z caller=logging.go:85 method=Health result=2 took=36.465µs +ts=2025-08-04T12:05:42Z caller=logging.go:85 method=Health result=2 took=25.414µs +ts=2025-08-04T12:05:45Z caller=logging.go:85 method=Health result=2 took=25.682µs +ts=2025-08-04T12:05:45Z caller=logging.go:85 method=Health result=2 took=32.347µs +ts=2025-08-04T12:05:48Z caller=logging.go:85 method=Health result=2 took=33.979µs +ts=2025-08-04T12:05:48Z caller=logging.go:85 method=Health result=2 took=26.419µs +ts=2025-08-04T12:05:51Z caller=logging.go:85 method=Health result=2 took=32.56µs +ts=2025-08-04T12:05:51Z caller=logging.go:85 method=Health result=2 took=28.238µs +ts=2025-08-04T12:05:54Z caller=logging.go:85 method=Health result=2 took=31.406µs +ts=2025-08-04T12:05:54Z caller=logging.go:85 method=Health result=2 took=34.804µs +ts=2025-08-04T12:05:57Z caller=logging.go:85 method=Health result=2 took=52.681µs +ts=2025-08-04T12:05:57Z caller=logging.go:85 method=Health result=2 took=29.84µs +ts=2025-08-04T12:06:00Z caller=logging.go:85 method=Health result=2 took=31.686µs +ts=2025-08-04T12:06:00Z caller=logging.go:85 method=Health result=2 took=21.267µs +ts=2025-08-04T12:06:03Z caller=logging.go:85 method=Health result=2 took=32.245µs +ts=2025-08-04T12:06:03Z caller=logging.go:85 method=Health result=2 took=25.368µs +ts=2025-08-04T12:06:06Z caller=logging.go:85 method=Health result=2 took=31.419µs +ts=2025-08-04T12:06:06Z caller=logging.go:85 method=Health result=2 took=27.054µs +ts=2025-08-04T12:06:09Z caller=logging.go:85 method=Health result=2 took=31.306µs +ts=2025-08-04T12:06:09Z caller=logging.go:85 method=Health result=2 took=25.306µs +ts=2025-08-04T12:06:12Z caller=logging.go:85 method=Health result=2 took=33.213µs +ts=2025-08-04T12:06:12Z caller=logging.go:85 method=Health result=2 took=43.913µs +ts=2025-08-04T12:06:15Z caller=logging.go:85 method=Health result=2 took=32.787µs +ts=2025-08-04T12:06:15Z caller=logging.go:85 method=Health result=2 took=25.318µs +ts=2025-08-04T12:06:18Z caller=logging.go:85 method=Health result=2 took=34.388µs +ts=2025-08-04T12:06:18Z caller=logging.go:85 method=Health result=2 took=25.728µs +ts=2025-08-04T12:06:21Z caller=logging.go:85 method=Health result=2 took=30.706µs +ts=2025-08-04T12:06:21Z caller=logging.go:85 method=Health result=2 took=29.45µs +ts=2025-08-04T12:06:24Z caller=logging.go:85 method=Health result=2 took=31.684µs +ts=2025-08-04T12:06:24Z caller=logging.go:85 method=Health result=2 took=27.35µs +ts=2025-08-04T12:06:27Z caller=logging.go:85 method=Health result=2 took=30.797µs +ts=2025-08-04T12:06:27Z caller=logging.go:85 method=Health result=2 took=25.423µs +ts=2025-08-04T12:06:30Z caller=logging.go:85 method=Health result=2 took=33.25µs +ts=2025-08-04T12:06:30Z caller=logging.go:85 method=Health result=2 took=25.4µs +ts=2025-08-04T12:06:33Z caller=logging.go:85 method=Health result=2 took=32.909µs +ts=2025-08-04T12:06:33Z caller=logging.go:85 method=Health result=2 took=25.397µs +ts=2025-08-04T12:06:36Z caller=logging.go:85 method=Health result=2 took=32.808µs +ts=2025-08-04T12:06:36Z caller=logging.go:85 method=Health result=2 took=25.073µs +ts=2025-08-04T12:06:39Z caller=logging.go:85 method=Health result=2 took=34.494µs +ts=2025-08-04T12:06:39Z caller=logging.go:85 method=Health result=2 took=27.747µs +ts=2025-08-04T12:06:42Z caller=logging.go:85 method=Health result=2 took=31.988µs +ts=2025-08-04T12:06:42Z caller=logging.go:85 method=Health result=2 took=25.822µs +ts=2025-08-04T12:06:45Z caller=logging.go:85 method=Health result=2 took=33.307µs +ts=2025-08-04T12:06:45Z caller=logging.go:85 method=Health result=2 took=25.178µs +ts=2025-08-04T12:06:48Z caller=logging.go:85 method=Health result=2 took=33.56µs +ts=2025-08-04T12:06:48Z caller=logging.go:85 method=Health result=2 took=25.486µs +ts=2025-08-04T12:06:51Z caller=logging.go:85 method=Health result=2 took=35.178µs +ts=2025-08-04T12:06:51Z caller=logging.go:85 method=Health result=2 took=23.251µs +ts=2025-08-04T12:06:54Z caller=logging.go:85 method=Health result=2 took=41.056µs +ts=2025-08-04T12:06:54Z caller=logging.go:85 method=Health result=2 took=43.964µs +ts=2025-08-04T12:06:57Z caller=logging.go:85 method=Health result=2 took=37.46µs +ts=2025-08-04T12:06:57Z caller=logging.go:85 method=Health result=2 took=27.444µs +ts=2025-08-04T12:07:00Z caller=logging.go:85 method=Health result=2 took=31.639µs +ts=2025-08-04T12:07:00Z caller=logging.go:85 method=Health result=2 took=27.903µs +ts=2025-08-04T12:07:03Z caller=logging.go:85 method=Health result=2 took=43.292µs +ts=2025-08-04T12:07:03Z caller=logging.go:85 method=Health result=2 took=32.486µs +ts=2025-08-04T12:07:06Z caller=logging.go:85 method=Health result=2 took=31.161µs +ts=2025-08-04T12:07:06Z caller=logging.go:85 method=Health result=2 took=25.11µs +ts=2025-08-04T12:07:09Z caller=logging.go:85 method=Health result=2 took=36.278µs +ts=2025-08-04T12:07:09Z caller=logging.go:85 method=Health result=2 took=25.179µs +ts=2025-08-04T12:07:12Z caller=logging.go:85 method=Health result=2 took=31.627µs +ts=2025-08-04T12:07:12Z caller=logging.go:85 method=Health result=2 took=24.317µs +ts=2025-08-04T12:07:15Z caller=logging.go:85 method=Health result=2 took=25.172µs +ts=2025-08-04T12:07:15Z caller=logging.go:85 method=Health result=2 took=32.56µs +ts=2025-08-04T12:07:18Z caller=logging.go:85 method=Health result=2 took=24.986µs +ts=2025-08-04T12:07:18Z caller=logging.go:85 method=Health result=2 took=56.946µs +ts=2025-08-04T12:07:21Z caller=logging.go:85 method=Health result=2 took=32.556µs +ts=2025-08-04T12:07:21Z caller=logging.go:85 method=Health result=2 took=25.938µs +ts=2025-08-04T12:07:24Z caller=logging.go:85 method=Health result=2 took=34.023µs +ts=2025-08-04T12:07:24Z caller=logging.go:85 method=Health result=2 took=25.265µs +ts=2025-08-04T12:07:27Z caller=logging.go:85 method=Health result=2 took=25.606µs +ts=2025-08-04T12:07:27Z caller=logging.go:85 method=Health result=2 took=43.629µs +ts=2025-08-04T12:07:30Z caller=logging.go:85 method=Health result=2 took=32.475µs +ts=2025-08-04T12:07:30Z caller=logging.go:85 method=Health result=2 took=27.03µs +ts=2025-08-04T12:07:33Z caller=logging.go:85 method=Health result=2 took=34.113µs +ts=2025-08-04T12:07:33Z caller=logging.go:85 method=Health result=2 took=25.604µs +ts=2025-08-04T12:07:36Z caller=logging.go:85 method=Health result=2 took=31.114µs +ts=2025-08-04T12:07:36Z caller=logging.go:85 method=Health result=2 took=43.754µs +ts=2025-08-04T12:07:39Z caller=logging.go:85 method=Health result=2 took=32.614µs +ts=2025-08-04T12:07:39Z caller=logging.go:85 method=Health result=2 took=25.52µs +ts=2025-08-04T12:07:42Z caller=logging.go:85 method=Health result=2 took=32.794µs +ts=2025-08-04T12:07:42Z caller=logging.go:85 method=Health result=2 took=24.479µs +ts=2025-08-04T12:07:45Z caller=logging.go:85 method=Health result=2 took=32.733µs +ts=2025-08-04T12:07:45Z caller=logging.go:85 method=Health result=2 took=14.206µs +ts=2025-08-04T12:07:48Z caller=logging.go:85 method=Health result=2 took=32.938µs +ts=2025-08-04T12:07:48Z caller=logging.go:85 method=Health result=2 took=25.059µs +ts=2025-08-04T12:07:51Z caller=logging.go:85 method=Health result=2 took=41.795µs +ts=2025-08-04T12:07:51Z caller=logging.go:85 method=Health result=2 took=25.573µs +ts=2025-08-04T12:07:54Z caller=logging.go:85 method=Health result=2 took=49.48µs +ts=2025-08-04T12:07:54Z caller=logging.go:85 method=Health result=2 took=38.761µs +ts=2025-08-04T12:07:57Z caller=logging.go:85 method=Health result=2 took=31.552µs +ts=2025-08-04T12:07:57Z caller=logging.go:85 method=Health result=2 took=25.45µs +ts=2025-08-04T12:08:00Z caller=logging.go:85 method=Health result=2 took=32.96µs +ts=2025-08-04T12:08:00Z caller=logging.go:85 method=Health result=2 took=25.148µs +ts=2025-08-04T12:08:03Z caller=logging.go:85 method=Health result=2 took=41.819µs +ts=2025-08-04T12:08:03Z caller=logging.go:85 method=Health result=2 took=41.425µs +ts=2025-08-04T12:08:06Z caller=logging.go:85 method=Health result=2 took=31.621µs +ts=2025-08-04T12:08:06Z caller=logging.go:85 method=Health result=2 took=26.213µs +ts=2025-08-04T12:08:09Z caller=logging.go:85 method=Health result=2 took=32.385µs +ts=2025-08-04T12:08:09Z caller=logging.go:85 method=Health result=2 took=25.786µs +ts=2025-08-04T12:08:12Z caller=logging.go:85 method=Health result=2 took=42.23µs +ts=2025-08-04T12:08:12Z caller=logging.go:85 method=Health result=2 took=49.916µs +ts=2025-08-04T12:08:15Z caller=logging.go:85 method=Health result=2 took=34.494µs +ts=2025-08-04T12:08:15Z caller=logging.go:85 method=Health result=2 took=26.276µs +ts=2025-08-04T12:08:18Z caller=logging.go:85 method=Health result=2 took=31.025µs +ts=2025-08-04T12:08:18Z caller=logging.go:85 method=Health result=2 took=25.058µs +ts=2025-08-04T12:08:21Z caller=logging.go:85 method=Health result=2 took=31.185µs +ts=2025-08-04T12:08:21Z caller=logging.go:85 method=Health result=2 took=41.297µs +ts=2025-08-04T12:08:24Z caller=logging.go:85 method=Health result=2 took=44.544µs +ts=2025-08-04T12:08:24Z caller=logging.go:85 method=Health result=2 took=28.401µs +ts=2025-08-04T12:08:27Z caller=logging.go:85 method=Health result=2 took=31.925µs +ts=2025-08-04T12:08:27Z caller=logging.go:85 method=Health result=2 took=26.975µs +ts=2025-08-04T12:08:30Z caller=logging.go:85 method=Health result=2 took=33.052µs +ts=2025-08-04T12:08:30Z caller=logging.go:85 method=Health result=2 took=38.316µs +ts=2025-08-04T12:08:33Z caller=logging.go:85 method=Health result=2 took=32.471µs +ts=2025-08-04T12:08:33Z caller=logging.go:85 method=Health result=2 took=27.222µs +ts=2025-08-04T12:08:36Z caller=logging.go:85 method=Health result=2 took=25.485µs +ts=2025-08-04T12:08:36Z caller=logging.go:85 method=Health result=2 took=32.766µs +ts=2025-08-04T12:08:39Z caller=logging.go:85 method=Health result=2 took=32.39µs +ts=2025-08-04T12:08:39Z caller=logging.go:85 method=Health result=2 took=26.145µs +ts=2025-08-04T12:08:42Z caller=logging.go:85 method=Health result=2 took=32.594µs +ts=2025-08-04T12:08:42Z caller=logging.go:85 method=Health result=2 took=24.97µs +ts=2025-08-04T12:08:45Z caller=logging.go:85 method=Health result=2 took=43.368µs +ts=2025-08-04T12:08:45Z caller=logging.go:85 method=Health result=2 took=20.749µs +ts=2025-08-04T12:08:48Z caller=logging.go:85 method=Health result=2 took=31.913µs +ts=2025-08-04T12:08:48Z caller=logging.go:85 method=Health result=2 took=164.293µs +ts=2025-08-04T12:08:51Z caller=logging.go:85 method=Health result=2 took=32.772µs +ts=2025-08-04T12:08:51Z caller=logging.go:85 method=Health result=2 took=25.351µs +ts=2025-08-04T12:08:54Z caller=logging.go:85 method=Health result=2 took=31.395µs +ts=2025-08-04T12:08:54Z caller=logging.go:85 method=Health result=2 took=25.373µs +ts=2025-08-04T12:08:57Z caller=logging.go:85 method=Health result=2 took=33.068µs +ts=2025-08-04T12:08:57Z caller=logging.go:85 method=Health result=2 took=24.82µs +ts=2025-08-04T12:09:00Z caller=logging.go:85 method=Health result=2 took=41.758µs +ts=2025-08-04T12:09:00Z caller=logging.go:85 method=Health result=2 took=32.479µs +ts=2025-08-04T12:09:03Z caller=logging.go:85 method=Health result=2 took=32.641µs +ts=2025-08-04T12:09:03Z caller=logging.go:85 method=Health result=2 took=27.57µs +ts=2025-08-04T12:09:06Z caller=logging.go:85 method=Health result=2 took=29.375µs +ts=2025-08-04T12:09:06Z caller=logging.go:85 method=Health result=2 took=25.711µs +ts=2025-08-04T12:09:09Z caller=logging.go:85 method=Health result=2 took=32.522µs +ts=2025-08-04T12:09:09Z caller=logging.go:85 method=Health result=2 took=25.617µs +ts=2025-08-04T12:09:12Z caller=logging.go:85 method=Health result=2 took=41.085µs +ts=2025-08-04T12:09:12Z caller=logging.go:85 method=Health result=2 took=34.472µs +ts=2025-08-04T12:09:15Z caller=logging.go:85 method=Health result=2 took=32.34µs +ts=2025-08-04T12:09:15Z caller=logging.go:85 method=Health result=2 took=24.941µs +ts=2025-08-04T12:09:18Z caller=logging.go:85 method=Health result=2 took=37.695µs +ts=2025-08-04T12:09:18Z caller=logging.go:85 method=Health result=2 took=27.416µs +ts=2025-08-04T12:09:21Z caller=logging.go:85 method=Health result=2 took=41.791µs +ts=2025-08-04T12:09:21Z caller=logging.go:85 method=Health result=2 took=41.784µs +ts=2025-08-04T12:09:24Z caller=logging.go:85 method=Health result=2 took=31.121µs +ts=2025-08-04T12:09:24Z caller=logging.go:85 method=Health result=2 took=103.876µs +ts=2025-08-04T12:09:27Z caller=logging.go:85 method=Health result=2 took=33.031µs +ts=2025-08-04T12:09:27Z caller=logging.go:85 method=Health result=2 took=25.702µs +ts=2025-08-04T12:09:30Z caller=logging.go:85 method=Health result=2 took=33.19µs +ts=2025-08-04T12:09:30Z caller=logging.go:85 method=Health result=2 took=26.467µs +ts=2025-08-04T12:09:33Z caller=logging.go:85 method=Health result=2 took=31.65µs +ts=2025-08-04T12:09:33Z caller=logging.go:85 method=Health result=2 took=25.685µs +ts=2025-08-04T12:09:36Z caller=logging.go:85 method=Health result=2 took=33.24µs +ts=2025-08-04T12:09:36Z caller=logging.go:85 method=Health result=2 took=25.166µs +ts=2025-08-04T12:09:39Z caller=logging.go:85 method=Health result=2 took=40.996µs +ts=2025-08-04T12:09:39Z caller=logging.go:85 method=Health result=2 took=38.148µs +ts=2025-08-04T12:09:42Z caller=logging.go:85 method=Health result=2 took=30.981µs +ts=2025-08-04T12:09:42Z caller=logging.go:85 method=Health result=2 took=25.642µs +ts=2025-08-04T12:09:45Z caller=logging.go:85 method=Health result=2 took=41.178µs +ts=2025-08-04T12:09:45Z caller=logging.go:85 method=Health result=2 took=32.812µs +ts=2025-08-04T12:09:48Z caller=logging.go:85 method=Health result=2 took=32.21µs +ts=2025-08-04T12:09:48Z caller=logging.go:85 method=Health result=2 took=39.709µs +ts=2025-08-04T12:09:51Z caller=logging.go:85 method=Health result=2 took=41.432µs +ts=2025-08-04T12:09:51Z caller=logging.go:85 method=Health result=2 took=137.256µs +ts=2025-08-04T12:09:54Z caller=logging.go:85 method=Health result=2 took=32.398µs +ts=2025-08-04T12:09:54Z caller=logging.go:85 method=Health result=2 took=25.358µs +ts=2025-08-04T12:09:57Z caller=logging.go:85 method=Health result=2 took=30.659µs +ts=2025-08-04T12:09:57Z caller=logging.go:85 method=Health result=2 took=25.133µs +ts=2025-08-04T12:10:00Z caller=logging.go:85 method=Health result=2 took=43.179µs +ts=2025-08-04T12:10:00Z caller=logging.go:85 method=Health result=2 took=49.223µs +ts=2025-08-04T12:10:03Z caller=logging.go:85 method=Health result=2 took=41.598µs +ts=2025-08-04T12:10:03Z caller=logging.go:85 method=Health result=2 took=61.231µs +ts=2025-08-04T12:10:06Z caller=logging.go:85 method=Health result=2 took=35.483µs +ts=2025-08-04T12:10:06Z caller=logging.go:85 method=Health result=2 took=38.9µs +ts=2025-08-04T12:10:09Z caller=logging.go:85 method=Health result=2 took=32.816µs +ts=2025-08-04T12:10:09Z caller=logging.go:85 method=Health result=2 took=24.969µs +ts=2025-08-04T12:10:12Z caller=logging.go:85 method=Health result=2 took=31.869µs +ts=2025-08-04T12:10:12Z caller=logging.go:85 method=Health result=2 took=25.329µs +ts=2025-08-04T12:10:15Z caller=logging.go:85 method=Health result=2 took=33.726µs +ts=2025-08-04T12:10:15Z caller=logging.go:85 method=Health result=2 took=25.147µs +ts=2025-08-04T12:10:18Z caller=logging.go:85 method=Health result=2 took=47.645µs +ts=2025-08-04T12:10:18Z caller=logging.go:85 method=Health result=2 took=25.151µs +ts=2025-08-04T12:10:21Z caller=logging.go:85 method=Health result=2 took=31.75µs +ts=2025-08-04T12:10:21Z caller=logging.go:85 method=Health result=2 took=16.652µs +ts=2025-08-04T12:10:24Z caller=logging.go:85 method=Health result=2 took=32.727µs +ts=2025-08-04T12:10:24Z caller=logging.go:85 method=Health result=2 took=25.173µs +ts=2025-08-04T12:10:27Z caller=logging.go:85 method=Health result=2 took=30.199µs +ts=2025-08-04T12:10:27Z caller=logging.go:85 method=Health result=2 took=38.714µs +ts=2025-08-04T12:10:30Z caller=logging.go:85 method=Health result=2 took=32.718µs +ts=2025-08-04T12:10:30Z caller=logging.go:85 method=Health result=2 took=29.93µs +ts=2025-08-04T12:10:33Z caller=logging.go:85 method=Health result=2 took=33.413µs +ts=2025-08-04T12:10:33Z caller=logging.go:85 method=Health result=2 took=26.096µs +ts=2025-08-04T12:10:36Z caller=logging.go:85 method=Health result=2 took=42.901µs +ts=2025-08-04T12:10:36Z caller=logging.go:85 method=Health result=2 took=42.298µs +ts=2025-08-04T12:10:39Z caller=logging.go:85 method=Health result=2 took=32.211µs +ts=2025-08-04T12:10:39Z caller=logging.go:85 method=Health result=2 took=45.933µs +ts=2025-08-04T12:10:42Z caller=logging.go:85 method=Health result=2 took=41.946µs +ts=2025-08-04T12:10:42Z caller=logging.go:85 method=Health result=2 took=25.879µs +ts=2025-08-04T12:10:45Z caller=logging.go:85 method=Health result=2 took=32.166µs +ts=2025-08-04T12:10:45Z caller=logging.go:85 method=Health result=2 took=24.893µs +ts=2025-08-04T12:10:48Z caller=logging.go:85 method=Health result=2 took=33.25µs +ts=2025-08-04T12:10:48Z caller=logging.go:85 method=Health result=2 took=24.805µs +ts=2025-08-04T12:10:51Z caller=logging.go:85 method=Health result=2 took=33.198µs +ts=2025-08-04T12:10:51Z caller=logging.go:85 method=Health result=2 took=26.197µs +ts=2025-08-04T12:10:54Z caller=logging.go:85 method=Health result=2 took=45.22µs +ts=2025-08-04T12:10:54Z caller=logging.go:85 method=Health result=2 took=60.919µs +ts=2025-08-04T12:10:57Z caller=logging.go:85 method=Health result=2 took=32.354µs +ts=2025-08-04T12:10:57Z caller=logging.go:85 method=Health result=2 took=29.125µs +ts=2025-08-04T12:11:00Z caller=logging.go:85 method=Health result=2 took=31.487µs +ts=2025-08-04T12:11:00Z caller=logging.go:85 method=Health result=2 took=27.252µs +ts=2025-08-04T12:11:03Z caller=logging.go:85 method=Health result=2 took=42.016µs +ts=2025-08-04T12:11:03Z caller=logging.go:85 method=Health result=2 took=31.009µs +ts=2025-08-04T12:11:06Z caller=logging.go:85 method=Health result=2 took=40.787µs +ts=2025-08-04T12:11:06Z caller=logging.go:85 method=Health result=2 took=35.64µs +ts=2025-08-04T12:11:09Z caller=logging.go:85 method=Health result=2 took=33.03µs +ts=2025-08-04T12:11:09Z caller=logging.go:85 method=Health result=2 took=25.998µs +ts=2025-08-04T12:11:12Z caller=logging.go:85 method=Health result=2 took=31.369µs +ts=2025-08-04T12:11:12Z caller=logging.go:85 method=Health result=2 took=25.563µs +ts=2025-08-04T12:11:15Z caller=logging.go:85 method=Health result=2 took=31.628µs +ts=2025-08-04T12:11:15Z caller=logging.go:85 method=Health result=2 took=45.043µs +ts=2025-08-04T12:11:18Z caller=logging.go:85 method=Health result=2 took=49.284µs +ts=2025-08-04T12:11:18Z caller=logging.go:85 method=Health result=2 took=42.564µs +ts=2025-08-04T12:11:21Z caller=logging.go:85 method=Health result=2 took=33.341µs +ts=2025-08-04T12:11:21Z caller=logging.go:85 method=Health result=2 took=29.852µs +ts=2025-08-04T12:11:24Z caller=logging.go:85 method=Health result=2 took=32.639µs +ts=2025-08-04T12:11:24Z caller=logging.go:85 method=Health result=2 took=25.295µs +ts=2025-08-04T12:11:27Z caller=logging.go:85 method=Health result=2 took=40.713µs +ts=2025-08-04T12:11:27Z caller=logging.go:85 method=Health result=2 took=28.737µs +ts=2025-08-04T12:11:30Z caller=logging.go:85 method=Health result=2 took=40.479µs +ts=2025-08-04T12:11:30Z caller=logging.go:85 method=Health result=2 took=27.597µs +ts=2025-08-04T12:11:33Z caller=logging.go:85 method=Health result=2 took=34.784µs +ts=2025-08-04T12:11:33Z caller=logging.go:85 method=Health result=2 took=21.181µs +ts=2025-08-04T12:11:36Z caller=logging.go:85 method=Health result=2 took=33.189µs +ts=2025-08-04T12:11:36Z caller=logging.go:85 method=Health result=2 took=32.683µs +ts=2025-08-04T12:11:39Z caller=logging.go:85 method=Health result=2 took=36.472µs +ts=2025-08-04T12:11:39Z caller=logging.go:85 method=Health result=2 took=15.162µs +ts=2025-08-04T12:11:42Z caller=logging.go:85 method=Health result=2 took=41.797µs +ts=2025-08-04T12:11:42Z caller=logging.go:85 method=Health result=2 took=41.798µs +ts=2025-08-04T12:11:45Z caller=logging.go:85 method=Health result=2 took=33.532µs +ts=2025-08-04T12:11:45Z caller=logging.go:85 method=Health result=2 took=26.623µs +ts=2025-08-04T12:11:48Z caller=logging.go:85 method=Health result=2 took=34.554µs +ts=2025-08-04T12:11:48Z caller=logging.go:85 method=Health result=2 took=25.288µs +ts=2025-08-04T12:11:51Z caller=logging.go:85 method=Health result=2 took=31.832µs +ts=2025-08-04T12:11:51Z caller=logging.go:85 method=Health result=2 took=27.66µs +ts=2025-08-04T12:11:54Z caller=logging.go:85 method=Health result=2 took=33.688µs +ts=2025-08-04T12:11:54Z caller=logging.go:85 method=Health result=2 took=24.951µs +ts=2025-08-04T12:11:57Z caller=logging.go:85 method=Health result=2 took=36.41µs +ts=2025-08-04T12:11:57Z caller=logging.go:85 method=Health result=2 took=32.492µs +ts=2025-08-04T12:12:00Z caller=logging.go:85 method=Health result=2 took=32.685µs +ts=2025-08-04T12:12:00Z caller=logging.go:85 method=Health result=2 took=26.656µs +ts=2025-08-04T12:12:03Z caller=logging.go:85 method=Health result=2 took=55.077µs +ts=2025-08-04T12:12:03Z caller=logging.go:85 method=Health result=2 took=46.741µs +ts=2025-08-04T12:12:06Z caller=logging.go:85 method=Health result=2 took=32.606µs +ts=2025-08-04T12:12:06Z caller=logging.go:85 method=Health result=2 took=25.595µs +ts=2025-08-04T12:12:09Z caller=logging.go:85 method=Health result=2 took=34.194µs +ts=2025-08-04T12:12:09Z caller=logging.go:85 method=Health result=2 took=22.71µs +ts=2025-08-04T12:12:12Z caller=logging.go:85 method=Health result=2 took=58.966µs +ts=2025-08-04T12:12:12Z caller=logging.go:85 method=Health result=2 took=24.849µs +ts=2025-08-04T12:12:15Z caller=logging.go:85 method=Health result=2 took=30.9µs +ts=2025-08-04T12:12:15Z caller=logging.go:85 method=Health result=2 took=24.441µs +ts=2025-08-04T12:12:18Z caller=logging.go:85 method=Health result=2 took=31.607µs +ts=2025-08-04T12:12:18Z caller=logging.go:85 method=Health result=2 took=26.474µs +ts=2025-08-04T12:12:21Z caller=logging.go:85 method=Health result=2 took=31.247µs +ts=2025-08-04T12:12:21Z caller=logging.go:85 method=Health result=2 took=25.493µs +ts=2025-08-04T12:12:24Z caller=logging.go:85 method=Health result=2 took=31.497µs +ts=2025-08-04T12:12:24Z caller=logging.go:85 method=Health result=2 took=25.348µs +ts=2025-08-04T12:12:27Z caller=logging.go:85 method=Health result=2 took=33.185µs +ts=2025-08-04T12:12:27Z caller=logging.go:85 method=Health result=2 took=25.229µs +ts=2025-08-04T12:12:30Z caller=logging.go:85 method=Health result=2 took=32.582µs +ts=2025-08-04T12:12:30Z caller=logging.go:85 method=Health result=2 took=25.327µs +ts=2025-08-04T12:12:33Z caller=logging.go:85 method=Health result=2 took=30.902µs +ts=2025-08-04T12:12:33Z caller=logging.go:85 method=Health result=2 took=25.538µs +ts=2025-08-04T12:12:36Z caller=logging.go:85 method=Health result=2 took=31.167µs +ts=2025-08-04T12:12:36Z caller=logging.go:85 method=Health result=2 took=25.797µs +ts=2025-08-04T12:12:39Z caller=logging.go:85 method=Health result=2 took=32.427µs +ts=2025-08-04T12:12:39Z caller=logging.go:85 method=Health result=2 took=26.864µs +ts=2025-08-04T12:12:42Z caller=logging.go:85 method=Health result=2 took=33.084µs +ts=2025-08-04T12:12:42Z caller=logging.go:85 method=Health result=2 took=38.615µs +ts=2025-08-04T12:12:45Z caller=logging.go:85 method=Health result=2 took=31.506µs +ts=2025-08-04T12:12:45Z caller=logging.go:85 method=Health result=2 took=25.407µs +ts=2025-08-04T12:12:48Z caller=logging.go:85 method=Health result=2 took=32.142µs +ts=2025-08-04T12:12:48Z caller=logging.go:85 method=Health result=2 took=24.995µs +ts=2025-08-04T12:12:51Z caller=logging.go:85 method=Health result=2 took=35.541µs +ts=2025-08-04T12:12:51Z caller=logging.go:85 method=Health result=2 took=25.541µs +ts=2025-08-04T12:12:54Z caller=logging.go:85 method=Health result=2 took=33.278µs +ts=2025-08-04T12:12:54Z caller=logging.go:85 method=Health result=2 took=25.804µs +ts=2025-08-04T12:12:57Z caller=logging.go:85 method=Health result=2 took=49.715µs +ts=2025-08-04T12:12:57Z caller=logging.go:85 method=Health result=2 took=27.561µs +ts=2025-08-04T12:13:00Z caller=logging.go:85 method=Health result=2 took=37.672µs +ts=2025-08-04T12:13:00Z caller=logging.go:85 method=Health result=2 took=25.713µs +ts=2025-08-04T12:13:03Z caller=logging.go:85 method=Health result=2 took=44.495µs +ts=2025-08-04T12:13:03Z caller=logging.go:85 method=Health result=2 took=29.893µs +ts=2025-08-04T12:13:06Z caller=logging.go:85 method=Health result=2 took=34.79µs +ts=2025-08-04T12:13:06Z caller=logging.go:85 method=Health result=2 took=34.98µs +ts=2025-08-04T12:13:09Z caller=logging.go:85 method=Health result=2 took=31.033µs +ts=2025-08-04T12:13:09Z caller=logging.go:85 method=Health result=2 took=25.251µs +ts=2025-08-04T12:13:12Z caller=logging.go:85 method=Health result=2 took=34.248µs +ts=2025-08-04T12:13:12Z caller=logging.go:85 method=Health result=2 took=26.358µs +ts=2025-08-04T12:13:15Z caller=logging.go:85 method=Health result=2 took=32.479µs +ts=2025-08-04T12:13:15Z caller=logging.go:85 method=Health result=2 took=25.848µs +ts=2025-08-04T12:13:18Z caller=logging.go:85 method=Health result=2 took=30.756µs +ts=2025-08-04T12:13:18Z caller=logging.go:85 method=Health result=2 took=25.76µs +ts=2025-08-04T12:13:21Z caller=logging.go:85 method=Health result=2 took=37.396µs +ts=2025-08-04T12:13:21Z caller=logging.go:85 method=Health result=2 took=25.614µs +ts=2025-08-04T12:13:24Z caller=logging.go:85 method=Health result=2 took=31.666µs +ts=2025-08-04T12:13:24Z caller=logging.go:85 method=Health result=2 took=39.271µs +ts=2025-08-04T12:13:27Z caller=logging.go:85 method=Health result=2 took=32.854µs +ts=2025-08-04T12:13:27Z caller=logging.go:85 method=Health result=2 took=25.023µs +ts=2025-08-04T12:13:30Z caller=logging.go:85 method=Health result=2 took=32.259µs +ts=2025-08-04T12:13:30Z caller=logging.go:85 method=Health result=2 took=25.584µs +ts=2025-08-04T12:13:33Z caller=logging.go:85 method=Health result=2 took=32.026µs +ts=2025-08-04T12:13:33Z caller=logging.go:85 method=Health result=2 took=24.765µs +ts=2025-08-04T12:13:36Z caller=logging.go:85 method=Health result=2 took=47.008µs +ts=2025-08-04T12:13:36Z caller=logging.go:85 method=Health result=2 took=16.285µs +ts=2025-08-04T12:13:39Z caller=logging.go:85 method=Health result=2 took=44.014µs +ts=2025-08-04T12:13:39Z caller=logging.go:85 method=Health result=2 took=23.217µs +ts=2025-08-04T12:13:42Z caller=logging.go:85 method=Health result=2 took=30.815µs +ts=2025-08-04T12:13:42Z caller=logging.go:85 method=Health result=2 took=21.366µs +ts=2025-08-04T12:13:45Z caller=logging.go:85 method=Health result=2 took=31.87µs +ts=2025-08-04T12:13:45Z caller=logging.go:85 method=Health result=2 took=25.002µs +ts=2025-08-04T12:13:48Z caller=logging.go:85 method=Health result=2 took=32.743µs +ts=2025-08-04T12:13:48Z caller=logging.go:85 method=Health result=2 took=24.623µs +ts=2025-08-04T12:13:51Z caller=logging.go:85 method=Health result=2 took=33.958µs +ts=2025-08-04T12:13:51Z caller=logging.go:85 method=Health result=2 took=26.631µs +ts=2025-08-04T12:13:54Z caller=logging.go:85 method=Health result=2 took=31.337µs +ts=2025-08-04T12:13:54Z caller=logging.go:85 method=Health result=2 took=35.04µs +ts=2025-08-04T12:13:57Z caller=logging.go:85 method=Health result=2 took=30.915µs +ts=2025-08-04T12:13:57Z caller=logging.go:85 method=Health result=2 took=207.712µs +ts=2025-08-04T12:14:00Z caller=logging.go:85 method=Health result=2 took=42.959µs +ts=2025-08-04T12:14:00Z caller=logging.go:85 method=Health result=2 took=42.665µs +ts=2025-08-04T12:14:03Z caller=logging.go:85 method=Health result=2 took=32.051µs +ts=2025-08-04T12:14:03Z caller=logging.go:85 method=Health result=2 took=25.368µs +ts=2025-08-04T12:14:06Z caller=logging.go:85 method=Health result=2 took=32.04µs +ts=2025-08-04T12:14:06Z caller=logging.go:85 method=Health result=2 took=42.161µs +ts=2025-08-04T12:14:09Z caller=logging.go:85 method=Health result=2 took=32.678µs +ts=2025-08-04T12:14:09Z caller=logging.go:85 method=Health result=2 took=44.968µs +ts=2025-08-04T12:14:12Z caller=logging.go:85 method=Health result=2 took=31.105µs +ts=2025-08-04T12:14:12Z caller=logging.go:85 method=Health result=2 took=28.734µs +ts=2025-08-04T12:14:15Z caller=logging.go:85 method=Health result=2 took=33.14µs +ts=2025-08-04T12:14:15Z caller=logging.go:85 method=Health result=2 took=39.435µs +ts=2025-08-04T12:14:18Z caller=logging.go:85 method=Health result=2 took=31.334µs +ts=2025-08-04T12:14:18Z caller=logging.go:85 method=Health result=2 took=26.256µs +ts=2025-08-04T12:14:21Z caller=logging.go:85 method=Health result=2 took=31.255µs +ts=2025-08-04T12:14:21Z caller=logging.go:85 method=Health result=2 took=24.858µs +ts=2025-08-04T12:14:24Z caller=logging.go:85 method=Health result=2 took=31.034µs +ts=2025-08-04T12:14:24Z caller=logging.go:85 method=Health result=2 took=25.299µs +ts=2025-08-04T12:14:27Z caller=logging.go:85 method=Health result=2 took=33.149µs +ts=2025-08-04T12:14:27Z caller=logging.go:85 method=Health result=2 took=25.838µs +ts=2025-08-04T12:14:30Z caller=logging.go:85 method=Health result=2 took=33.383µs +ts=2025-08-04T12:14:30Z caller=logging.go:85 method=Health result=2 took=183.79µs +ts=2025-08-04T12:14:33Z caller=logging.go:85 method=Health result=2 took=30.755µs +ts=2025-08-04T12:14:33Z caller=logging.go:85 method=Health result=2 took=25.542µs +ts=2025-08-04T12:14:36Z caller=logging.go:85 method=Health result=2 took=31.988µs +ts=2025-08-04T12:14:36Z caller=logging.go:85 method=Health result=2 took=24.954µs +ts=2025-08-04T12:14:39Z caller=logging.go:85 method=Health result=2 took=34.376µs +ts=2025-08-04T12:14:39Z caller=logging.go:85 method=Health result=2 took=19.65µs +ts=2025-08-04T12:14:42Z caller=logging.go:85 method=Health result=2 took=31.322µs +ts=2025-08-04T12:14:42Z caller=logging.go:85 method=Health result=2 took=27.856µs +ts=2025-08-04T12:14:45Z caller=logging.go:85 method=Health result=2 took=42.024µs +ts=2025-08-04T12:14:45Z caller=logging.go:85 method=Health result=2 took=35.238µs +ts=2025-08-04T12:14:48Z caller=logging.go:85 method=Health result=2 took=38.026µs +ts=2025-08-04T12:14:48Z caller=logging.go:85 method=Health result=2 took=32.127µs +ts=2025-08-04T12:14:51Z caller=logging.go:85 method=Health result=2 took=44.997µs +ts=2025-08-04T12:14:51Z caller=logging.go:85 method=Health result=2 took=44.997µs +ts=2025-08-04T12:14:54Z caller=logging.go:85 method=Health result=2 took=32.44µs +ts=2025-08-04T12:14:54Z caller=logging.go:85 method=Health result=2 took=27.821µs +ts=2025-08-04T12:14:57Z caller=logging.go:85 method=Health result=2 took=32.053µs +ts=2025-08-04T12:14:57Z caller=logging.go:85 method=Health result=2 took=25.071µs +ts=2025-08-04T12:15:00Z caller=logging.go:85 method=Health result=2 took=43.879µs +ts=2025-08-04T12:15:00Z caller=logging.go:85 method=Health result=2 took=34.174µs +ts=2025-08-04T12:15:03Z caller=logging.go:85 method=Health result=2 took=32.884µs +ts=2025-08-04T12:15:03Z caller=logging.go:85 method=Health result=2 took=25.771µs +ts=2025-08-04T12:15:06Z caller=logging.go:85 method=Health result=2 took=31.378µs +ts=2025-08-04T12:15:06Z caller=logging.go:85 method=Health result=2 took=25.605µs +ts=2025-08-04T12:15:09Z caller=logging.go:85 method=Health result=2 took=42.874µs +ts=2025-08-04T12:15:09Z caller=logging.go:85 method=Health result=2 took=54.038µs +ts=2025-08-04T12:15:12Z caller=logging.go:85 method=Health result=2 took=33.296µs +ts=2025-08-04T12:15:12Z caller=logging.go:85 method=Health result=2 took=28.855µs +ts=2025-08-04T12:15:15Z caller=logging.go:85 method=Health result=2 took=31.441µs +ts=2025-08-04T12:15:15Z caller=logging.go:85 method=Health result=2 took=317.368µs +ts=2025-08-04T12:15:18Z caller=logging.go:85 method=Health result=2 took=25.324µs +ts=2025-08-04T12:15:18Z caller=logging.go:85 method=Health result=2 took=31.608µs +ts=2025-08-04T12:15:21Z caller=logging.go:85 method=Health result=2 took=31.802µs +ts=2025-08-04T12:15:21Z caller=logging.go:85 method=Health result=2 took=25.729µs +ts=2025-08-04T12:15:24Z caller=logging.go:85 method=Health result=2 took=35.893µs +ts=2025-08-04T12:15:24Z caller=logging.go:85 method=Health result=2 took=36.656µs +ts=2025-08-04T12:15:27Z caller=logging.go:85 method=Health result=2 took=33.364µs +ts=2025-08-04T12:15:27Z caller=logging.go:85 method=Health result=2 took=40.755µs +ts=2025-08-04T12:15:30Z caller=logging.go:85 method=Health result=2 took=32.694µs +ts=2025-08-04T12:15:30Z caller=logging.go:85 method=Health result=2 took=21.181µs +ts=2025-08-04T12:15:33Z caller=logging.go:85 method=Health result=2 took=32.007µs +ts=2025-08-04T12:15:33Z caller=logging.go:85 method=Health result=2 took=26.149µs +ts=2025-08-04T12:15:36Z caller=logging.go:85 method=Health result=2 took=32.085µs +ts=2025-08-04T12:15:36Z caller=logging.go:85 method=Health result=2 took=29.351µs +ts=2025-08-04T12:15:39Z caller=logging.go:85 method=Health result=2 took=25.148µs +ts=2025-08-04T12:15:39Z caller=logging.go:85 method=Health result=2 took=43.974µs +ts=2025-08-04T12:15:42Z caller=logging.go:85 method=Health result=2 took=33.867µs +ts=2025-08-04T12:15:42Z caller=logging.go:85 method=Health result=2 took=45.517µs +ts=2025-08-04T12:15:45Z caller=logging.go:85 method=Health result=2 took=132.32µs +ts=2025-08-04T12:15:45Z caller=logging.go:85 method=Health result=2 took=25.259µs +ts=2025-08-04T12:15:48Z caller=logging.go:85 method=Health result=2 took=31.441µs +ts=2025-08-04T12:15:48Z caller=logging.go:85 method=Health result=2 took=31.994µs +ts=2025-08-04T12:15:51Z caller=logging.go:85 method=Health result=2 took=42.589µs +ts=2025-08-04T12:15:51Z caller=logging.go:85 method=Health result=2 took=40.346µs +ts=2025-08-04T12:15:54Z caller=logging.go:85 method=Health result=2 took=30.667µs +ts=2025-08-04T12:15:54Z caller=logging.go:85 method=Health result=2 took=24.82µs +ts=2025-08-04T12:15:57Z caller=logging.go:85 method=Health result=2 took=43.359µs +ts=2025-08-04T12:15:57Z caller=logging.go:85 method=Health result=2 took=26.241µs diff --git a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml index a935f061b0..674e157cfe 100644 --- a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml @@ -11,5 +11,8 @@ expected_output: | evaluation: correctness: 1 + +# setup & teardown + tags: - easy diff --git a/tests/llm/utils/mock_toolset.py b/tests/llm/utils/mock_toolset.py index 9bedf69dcf..a39bb643fb 100644 --- a/tests/llm/utils/mock_toolset.py +++ b/tests/llm/utils/mock_toolset.py @@ -5,9 +5,9 @@ import os import re from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Type import urllib - +import threading from pydantic import BaseModel import pytest @@ -62,7 +62,7 @@ class MockMode(Enum): """Modes for mock tool execution.""" MOCK = "mock" # Use existing mock files - GENERATE = "generate" # Generate new mock files + GENERATE = "generate" # Generate new mock files and run tools in live mode LIVE = "live" # Use real tools without mocking @@ -188,6 +188,7 @@ def __init__(self, test_case_folder: str, add_params_to_filename: bool = True): self.test_case_folder = test_case_folder self.add_params_to_filename = add_params_to_filename self._mock_cache = None # Cache for loaded mocks + self._lock = threading.Lock() def _get_mock_file_path(self, tool_name: str, params: Dict) -> str: """Generate the path for a mock file.""" @@ -300,94 +301,84 @@ def clear_mocks_for_test(self, request: pytest.FixtureRequest) -> List[str]: return cleared_files - def has_mock_files(self, tool_name: str) -> bool: - """Check if any mock files exist for this tool.""" - # First check with glob pattern for efficiency - pattern = os.path.join(self.test_case_folder, f"{tool_name}*.txt") - if len(glob.glob(pattern)) > 0: - return True - - # Also check loaded mocks in case filename doesn't match tool name - all_mocks = self._load_all_mocks() - return tool_name in all_mocks and len(all_mocks[tool_name]) > 0 - def _load_all_mocks(self) -> Dict[str, List[ToolMock]]: """Load all mock files in the directory and organize by tool name.""" - if self._mock_cache is not None: - return self._mock_cache + with self._lock: + if self._mock_cache is not None: + return self._mock_cache - self._mock_cache = {} + self._mock_cache = {} - # Load all .txt files in the directory - for file_path in glob.glob(os.path.join(self.test_case_folder, "*.txt")): - try: - with open(file_path, "r") as f: - lines = f.readlines() - if len(lines) < 1: - continue + # Load all .txt files in the directory + for file_path in glob.glob(os.path.join(self.test_case_folder, "*.txt")): + try: + with open(file_path, "r") as f: + lines = f.readlines() + if len(lines) < 1: + continue - # Try to parse first line as JSON metadata - try: - first_line = lines[0].strip() - metadata = json.loads(first_line) - if "tool_name" not in metadata: + # Try to parse first line as JSON metadata + try: + first_line = lines[0].strip() + metadata = json.loads(first_line) + if "tool_name" not in metadata: + continue + except json.JSONDecodeError: + # Not a mock file, skip continue - except json.JSONDecodeError: - # Not a mock file, skip - continue - # This looks like a mock file, parse it - if len(lines) < 2: - continue + # This looks like a mock file, parse it + if len(lines) < 2: + continue - try: - structured_output = json.loads(lines[1].strip()) - except json.JSONDecodeError: - # Check if this is an old format mock file - # Old format: Line 1 = metadata JSON, Line 2+ = raw output - # New format: Line 1 = metadata JSON, Line 2 = structured output JSON, Line 3+ = raw output - logging.error( - f"Mock file {file_path} appears to be in old format (missing structured JSON on second line). " - f"The mock file format was updated to include structured tool output metadata. " - f"Old format: Line 1 = metadata JSON, Line 2+ = raw output. " - f"New format: Line 1 = metadata JSON, Line 2 = structured output JSON, Line 3+ = raw output. " - f"This change was introduced in PR https://github.com/robusta-dev/holmesgpt/pull/372. " - f"Please regenerate your mock files using --regenerate-all-mocks or manually update them to the new format." + try: + structured_output = json.loads(lines[1].strip()) + except json.JSONDecodeError: + # Check if this is an old format mock file + # Old format: Line 1 = metadata JSON, Line 2+ = raw output + # New format: Line 1 = metadata JSON, Line 2 = structured output JSON, Line 3+ = raw output + logging.error( + f"Mock file {file_path} appears to be in old format (missing structured JSON on second line). " + f"The mock file format was updated to include structured tool output metadata. " + f"Old format: Line 1 = metadata JSON, Line 2+ = raw output. " + f"New format: Line 1 = metadata JSON, Line 2 = structured output JSON, Line 3+ = raw output. " + f"This change was introduced in PR https://github.com/robusta-dev/holmesgpt/pull/372. " + f"Please regenerate your mock files using --regenerate-all-mocks or manually update them to the new format." + ) + raise MockDataCorruptedError( + f"Mock file {file_path} is in old format and needs to be updated (see PR #372)", + tool_name=metadata.get("tool_name", "unknown"), + ) from None + + content = "".join(lines[2:]) if len(lines) > 2 else None + if content is not None: + structured_output["data"] = content + + mock = ToolMock( + toolset_name=metadata.get("toolset_name", ""), + tool_name=metadata["tool_name"], + match_params=metadata.get("match_params"), + source_file=file_path, + return_value=StructuredToolResult(**structured_output), ) - raise MockDataCorruptedError( - f"Mock file {file_path} is in old format and needs to be updated (see PR #372)", - tool_name=metadata.get("tool_name", "unknown"), - ) from None - - content = "".join(lines[2:]) if len(lines) > 2 else None - if content is not None: - structured_output["data"] = content - - mock = ToolMock( - toolset_name=metadata.get("toolset_name", ""), - tool_name=metadata["tool_name"], - match_params=metadata.get("match_params"), - source_file=file_path, - return_value=StructuredToolResult(**structured_output), - ) - # Add to cache organized by tool name - tool_name = metadata["tool_name"] - if tool_name not in self._mock_cache: - self._mock_cache[tool_name] = [] - self._mock_cache[tool_name].append(mock) - logging.debug( - f"Loaded mock for {tool_name} from {file_path}: match_params={mock.match_params}" - ) + # Add to cache organized by tool name + tool_name = metadata["tool_name"] + if tool_name not in self._mock_cache: + self._mock_cache[tool_name] = [] + self._mock_cache[tool_name].append(mock) + logging.debug( + f"Loaded mock for {tool_name} from {file_path}: match_params={mock.match_params}" + ) - except MockDataError: - # Re-raise MockDataError types (including old format error) to propagate them - raise - except Exception as e: - logging.warning(f"Failed to load mock file {file_path}: {e}") - continue + except MockDataError: + # Re-raise MockDataError types (including old format error) to propagate them + raise + except Exception as e: + logging.warning(f"Failed to load mock file {file_path}: {e}") + continue - return self._mock_cache + return self._mock_cache class MockableToolWrapper(Tool): @@ -413,42 +404,46 @@ def __init__( self._toolset_name = toolset_name self._request = request - def _invoke(self, params: Dict) -> StructuredToolResult: - """Execute the tool based on the current mode.""" - if self._mode == MockMode.LIVE: - # Live mode: just call the real tool - logging.info(f"Calling live tool {self.name} with params: {params}") - return self._tool.invoke(params) - - elif self._mode == MockMode.MOCK: - # Mock mode: read from mock file - mock = self._file_manager.read_mock(self.name, params) - if not mock: - # Check if there are any mock files for this tool that might be in old format - pattern = os.path.join( - self._file_manager.test_case_folder, f"{self.name}*.txt" + def _call_live_invoke(self, params: Dict) -> StructuredToolResult: + """Call the tool in live mode.""" + logging.info(f"Calling live tool {self.name} with params: {params}") + return self._tool.invoke(params) + + def _call_mock_invoke(self, params: Dict): + # Mock mode: read from mock file + mock = self._file_manager.read_mock(self.name, params) + if not mock: + # Check if there are any mock files for this tool that might be in old format + pattern = os.path.join( + self._file_manager.test_case_folder, f"{self.name}*.txt" + ) + existing_files = glob.glob(pattern) + + if existing_files: + # There are mock files, but none matched - could be old format + error_msg = ( + f"No mock data found for tool '{self.name}' with params: {params}. " + f"Found {len(existing_files)} mock file(s) for this tool, but none matched the parameters. " + f"This could be due to mock files being in the old format (missing structured JSON on line 2). " + f"See PR https://github.com/robusta-dev/holmesgpt/pull/372 for format details." + ) + else: + error_msg = ( + f"No mock data found for tool '{self.name}' with params: {params}" ) - existing_files = glob.glob(pattern) - - if existing_files: - # There are mock files, but none matched - could be old format - error_msg = ( - f"No mock data found for tool '{self.name}' with params: {params}. " - f"Found {len(existing_files)} mock file(s) for this tool, but none matched the parameters. " - f"This could be due to mock files being in the old format (missing structured JSON on line 2). " - f"See PR https://github.com/robusta-dev/holmesgpt/pull/372 for format details." - ) - else: - error_msg = f"No mock data found for tool '{self.name}' with params: {params}" - raise MockDataNotFoundError(error_msg, tool_name=self.name) - return mock.return_value + raise MockDataNotFoundError(error_msg, tool_name=self.name) + return mock.return_value - elif self._mode == MockMode.GENERATE: - # Generate mode: call real tool and save result - logging.info(f"Generating mock for tool {self.name} with params: {params}") - result = self._tool.invoke(params) + def _invoke(self, params: Dict) -> StructuredToolResult: + """Execute the tool based on the current mode.""" + if self._mode == MockMode.GENERATE: + try: + return self._call_mock_invoke(params) + except MockDataNotFoundError as e: + logging.debug(str(e)) + result = self._call_live_invoke(params) # Write mock file mock_file_path = self._file_manager.write_mock( tool_name=self.name, @@ -465,7 +460,11 @@ def _invoke(self, params: Dict) -> StructuredToolResult: ) return result - + elif self._mode == MockMode.LIVE: + # Live mode: just call the real tool + return self.call_live_invoke(params) + elif self._mode == MockMode.MOCK: + return self._call_mock_invoke(params) else: raise ValueError(f"Unknown mock mode: {self._mode}") @@ -477,6 +476,8 @@ def get_parameterized_one_liner(self, params: Dict) -> str: class SimplifiedMockToolset(Toolset): """Simplified mock toolset for testing.""" + original_toolset_type: Type[Toolset] + def get_status(self): return ToolsetStatusEnum.ENABLED @@ -583,7 +584,7 @@ def _configure_toolsets( # Check prerequisites for enabled toolsets with timeout # Only check prerequisites in LIVE mode - for MOCK/GENERATE modes we don't need real connections if toolset.enabled: - if self.mode == MockMode.LIVE: + if self.mode == MockMode.LIVE or self.mode == MockMode.GENERATE: try: # TODO: add timeout toolset.check_prerequisites() @@ -632,6 +633,7 @@ def _wrap_enabled_toolsets(self): description=toolset.description, llm_instructions=toolset.llm_instructions, config=toolset.config, + original_toolset_type=type(toolset), ) mock_toolset.status = ToolsetStatusEnum.ENABLED self.toolsets[i] = mock_toolset diff --git a/tests/llm/utils/test_case_utils.py b/tests/llm/utils/test_case_utils.py index aca9dda14f..ba7ab67ba5 100644 --- a/tests/llm/utils/test_case_utils.py +++ b/tests/llm/utils/test_case_utils.py @@ -43,6 +43,9 @@ class Message(BaseModel): class HolmesTestCase(BaseModel): + class Config: + extra = "forbid" + id: str folder: str mocked_date: Optional[str] = None @@ -60,9 +63,15 @@ class HolmesTestCase(BaseModel): mock_policy: Optional[str] = ( "inherit" # Mock policy: always_mock, never_mock, or inherit ) + description: Optional[str] = None + generate_mocks: Optional[bool] = None + toolsets: Optional[Dict[str, Any]] = None class AskHolmesTestCase(HolmesTestCase, BaseModel): + class Config: + extra = "forbid" + user_prompt: Union[ str, List[str] ] # The user's question(s) to ask holmes - can be single string or array @@ -191,6 +200,10 @@ def load_test_cases(self) -> List[HolmesTestCase]: test_case = TypeAdapter(HealthCheckTestCase).validate_python( config_dict ) + else: + raise ValueError( + f"Test case folder {test_case_folder} is not a valid test case folder." + ) logging.debug(f"Successfully loaded test case {test_case_id}") except FileNotFoundError: From fc2f3167d361646f734ea57551393110934b46e9 Mon Sep 17 00:00:00 2001 From: Mohse Morad Date: Mon, 4 Aug 2025 15:50:59 +0300 Subject: [PATCH 04/11] Fix lock file --- poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index b66c85047d..c2d7b80f0d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4735,4 +4735,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "cfb3a97e3bcba6cab324468ef3a91616c198b4e51d8276b2cbfd89d360d1c70b" +content-hash = "a497b485263526b24ef319eec212edf5602a2a7335182093b7c448258d445876" From dea9011bef3a6fa9514625895878233937f8fb29 Mon Sep 17 00:00:00 2001 From: Mohse Morad Date: Mon, 4 Aug 2025 16:06:27 +0300 Subject: [PATCH 05/11] Fix tests --- .../conversation_history.json | 0 ...ogssock-shop_catalogue-db-c948fd796-w4vzl_-3600.txt | 0 ...od_logssock-shop_catalogue-f7687cb4-zsngc_-3600.txt | 0 .../test_case.yaml | 6 +++++- .../toolsets.yaml | 0 tests/llm/utils/test_case_utils.py | 10 +++++----- 6 files changed, 10 insertions(+), 6 deletions(-) rename tests/llm/fixtures/test_ask_holmes/{91_calling_datadog => 93_calling_datadog}/conversation_history.json (100%) rename tests/llm/fixtures/test_ask_holmes/{91_calling_datadog => 93_calling_datadog}/fetch_pod_logssock-shop_catalogue-db-c948fd796-w4vzl_-3600.txt (100%) rename tests/llm/fixtures/test_ask_holmes/{91_calling_datadog => 93_calling_datadog}/fetch_pod_logssock-shop_catalogue-f7687cb4-zsngc_-3600.txt (100%) rename tests/llm/fixtures/test_ask_holmes/{91_calling_datadog => 93_calling_datadog}/test_case.yaml (80%) rename tests/llm/fixtures/test_ask_holmes/{91_calling_datadog => 93_calling_datadog}/toolsets.yaml (100%) diff --git a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/conversation_history.json b/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/conversation_history.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/91_calling_datadog/conversation_history.json rename to tests/llm/fixtures/test_ask_holmes/93_calling_datadog/conversation_history.json diff --git a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/fetch_pod_logssock-shop_catalogue-db-c948fd796-w4vzl_-3600.txt b/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/fetch_pod_logssock-shop_catalogue-db-c948fd796-w4vzl_-3600.txt similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/91_calling_datadog/fetch_pod_logssock-shop_catalogue-db-c948fd796-w4vzl_-3600.txt rename to tests/llm/fixtures/test_ask_holmes/93_calling_datadog/fetch_pod_logssock-shop_catalogue-db-c948fd796-w4vzl_-3600.txt diff --git a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/fetch_pod_logssock-shop_catalogue-f7687cb4-zsngc_-3600.txt b/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/fetch_pod_logssock-shop_catalogue-f7687cb4-zsngc_-3600.txt similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/91_calling_datadog/fetch_pod_logssock-shop_catalogue-f7687cb4-zsngc_-3600.txt rename to tests/llm/fixtures/test_ask_holmes/93_calling_datadog/fetch_pod_logssock-shop_catalogue-f7687cb4-zsngc_-3600.txt diff --git a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/test_case.yaml similarity index 80% rename from tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml rename to tests/llm/fixtures/test_ask_holmes/93_calling_datadog/test_case.yaml index 674e157cfe..31612d6256 100644 --- a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/test_case.yaml @@ -3,7 +3,11 @@ description: | When holmes were asked to fetch logs, they were not able to do so. The main reason behind this are do to the conversation history and the lack of explicity saying that we enabled datadog logs toolset as part of the system prompt logging section. -user_prompt: "Can you retrieve the datadog logs for this?" +user_prompt: + - "Can you retrieve the datadog logs for this?" + - "Please show me the logs for this pod" + - "Show me logs" + expected_output: | Logs for catalogue-f7687cb4-zsngc pod show consistent health checks with no errors. diff --git a/tests/llm/fixtures/test_ask_holmes/91_calling_datadog/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/toolsets.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/91_calling_datadog/toolsets.yaml rename to tests/llm/fixtures/test_ask_holmes/93_calling_datadog/toolsets.yaml diff --git a/tests/llm/utils/test_case_utils.py b/tests/llm/utils/test_case_utils.py index ba7ab67ba5..61dbffce53 100644 --- a/tests/llm/utils/test_case_utils.py +++ b/tests/llm/utils/test_case_utils.py @@ -80,8 +80,8 @@ class Config: runbooks: Optional[Dict[str, Any]] = None # Optional runbook catalog override # Internal fields for variant handling - _variant_index: Optional[int] = None # Which variant this instance represents - _original_user_prompt: Optional[Union[str, List[str]]] = ( + variant_index: Optional[int] = None # Which variant this instance represents + original_user_prompt: Optional[Union[str, List[str]]] = ( None # Store original prompt(s) ) @@ -157,8 +157,8 @@ def load_test_cases(self) -> List[HolmesTestCase]: for i, prompt in enumerate(original_user_prompt): variant_config = config_dict.copy() variant_config["user_prompt"] = prompt + extra_prompt - variant_config["_variant_index"] = i - variant_config["_original_user_prompt"] = ( + variant_config["variant_index"] = i + variant_config["original_user_prompt"] = ( original_user_prompt ) variant_config["id"] = f"{test_case_id}[{i}]" @@ -172,7 +172,7 @@ def load_test_cases(self) -> List[HolmesTestCase]: config_dict["user_prompt"] = ( config_dict["user_prompt"] + extra_prompt ) - config_dict["_original_user_prompt"] = original_user_prompt + config_dict["original_user_prompt"] = original_user_prompt test_case = TypeAdapter(AskHolmesTestCase).validate_python( config_dict ) From c1edd62827b0d2f56ccdd08443585cf96a83aa5f Mon Sep 17 00:00:00 2001 From: Mohse Morad Date: Mon, 4 Aug 2025 16:13:57 +0300 Subject: [PATCH 06/11] Fix test type --- .../test_ask_holmes/93_calling_datadog/test_case.yaml | 1 + tests/llm/test_ask_holmes.py | 4 +++- tests/llm/utils/test_case_utils.py | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/test_case.yaml index 31612d6256..a8f09be7b3 100644 --- a/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/test_case.yaml @@ -15,6 +15,7 @@ expected_output: | evaluation: correctness: 1 +test_type: "server" # setup & teardown diff --git a/tests/llm/test_ask_holmes.py b/tests/llm/test_ask_holmes.py index d6b037fbb9..762917875b 100644 --- a/tests/llm/test_ask_holmes.py +++ b/tests/llm/test_ask_holmes.py @@ -301,7 +301,9 @@ def ask_holmes( llm=DefaultLLM(os.environ.get("MODEL", "gpt-4o"), tracer=tracer), ) - test_type = os.environ.get("ASK_HOLMES_TEST_TYPE", "cli").lower() + test_type = ( + test_case.test_type or os.environ.get("ASK_HOLMES_TEST_TYPE", "cli").lower() + ) if test_type == "cli": if test_case.conversation_history: pytest.skip("CLI mode does not support conversation history tests") diff --git a/tests/llm/utils/test_case_utils.py b/tests/llm/utils/test_case_utils.py index 61dbffce53..9fb26d78c9 100644 --- a/tests/llm/utils/test_case_utils.py +++ b/tests/llm/utils/test_case_utils.py @@ -84,6 +84,7 @@ class Config: original_user_prompt: Optional[Union[str, List[str]]] = ( None # Store original prompt(s) ) + test_type: Optional[str] = None # The type of test to run class InvestigateTestCase(HolmesTestCase, BaseModel): From eafc142fbe102022516939ee4d78ff466fdf95b5 Mon Sep 17 00:00:00 2001 From: Mohse Morad Date: Mon, 4 Aug 2025 16:55:44 +0300 Subject: [PATCH 07/11] Fix test type --- .../fixtures/test_ask_holmes/93_calling_datadog/test_case.yaml | 2 +- tests/llm/utils/mock_toolset.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/test_case.yaml index a8f09be7b3..a3c2581ea4 100644 --- a/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/test_case.yaml @@ -1,7 +1,7 @@ description: | A test case that simulates an issue real customer bumped into. When holmes were asked to fetch logs, they were not able to do so. - The main reason behind this are do to the conversation history and the lack of explicity saying that we enabled datadog logs toolset as part of the system prompt logging section. + The main reason behind this are due to the conversation history and the lack of explicity saying that we enabled datadog logs toolset as part of the system prompt logging section. user_prompt: - "Can you retrieve the datadog logs for this?" diff --git a/tests/llm/utils/mock_toolset.py b/tests/llm/utils/mock_toolset.py index a39bb643fb..7759e153f8 100644 --- a/tests/llm/utils/mock_toolset.py +++ b/tests/llm/utils/mock_toolset.py @@ -462,7 +462,7 @@ def _invoke(self, params: Dict) -> StructuredToolResult: return result elif self._mode == MockMode.LIVE: # Live mode: just call the real tool - return self.call_live_invoke(params) + return self._call_live_invoke(params) elif self._mode == MockMode.MOCK: return self._call_mock_invoke(params) else: From c74a414017586eb7b165b3f1a40ffab2355a5f94 Mon Sep 17 00:00:00 2001 From: Mohse Morad Date: Tue, 5 Aug 2025 10:27:34 +0300 Subject: [PATCH 08/11] Fix test type --- holmes/core/models.py | 1 + tests/llm/utils/test_case_utils.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/holmes/core/models.py b/holmes/core/models.py index 8eec8a7e4f..c610fbb252 100644 --- a/holmes/core/models.py +++ b/holmes/core/models.py @@ -126,6 +126,7 @@ class WorkloadHealthRequest(BaseModel): include_tool_call_results: bool = False prompt_template: str = "builtin://kubernetes_workload_ask.jinja2" model: Optional[str] = None + request: Any = None class ChatRequest(ChatRequestBaseModel): diff --git a/tests/llm/utils/test_case_utils.py b/tests/llm/utils/test_case_utils.py index 767cc52b46..f4954be499 100644 --- a/tests/llm/utils/test_case_utils.py +++ b/tests/llm/utils/test_case_utils.py @@ -91,6 +91,7 @@ class InvestigateTestCase(HolmesTestCase, BaseModel): issue_data: Optional[Dict] resource_instructions: Optional[ResourceInstructions] expected_sections: Optional[Dict[str, Union[List[str], bool]]] = None + request: Any = None class HealthCheckTestCase(HolmesTestCase, BaseModel): @@ -122,7 +123,7 @@ def load_investigate_test_cases(self) -> List[InvestigateTestCase]: return cast(List[InvestigateTestCase], self.load_test_cases()) def load_ask_holmes_test_cases(self) -> List[AskHolmesTestCase]: - return self.load_test_cases() + return cast(List[AskHolmesTestCase], self.load_test_cases()) def load_test_cases(self) -> List[HolmesTestCase]: test_cases: List[HolmesTestCase] = [] From 9857607757da664564af04d1b02c6e7344a76bda Mon Sep 17 00:00:00 2001 From: Mohse Morad Date: Tue, 5 Aug 2025 11:30:59 +0300 Subject: [PATCH 09/11] Fix test type --- docs/development/evals/writing.md | 4 ++-- .../test_investigate/03_cpu_throttling/test_case.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/development/evals/writing.md b/docs/development/evals/writing.md index 400f7dccf4..ffee935170 100644 --- a/docs/development/evals/writing.md +++ b/docs/development/evals/writing.md @@ -303,8 +303,8 @@ spec: ```yaml user_prompt: 'How is the test-nginx deployment performing?' -before-test: kubectl apply -f manifest.yaml -after-test: kubectl delete -f manifest.yaml +before_test: kubectl apply -f manifest.yaml +after_test: kubectl delete -f manifest.yaml # ... rest of configuration ``` diff --git a/tests/llm/fixtures/test_investigate/03_cpu_throttling/test_case.yaml b/tests/llm/fixtures/test_investigate/03_cpu_throttling/test_case.yaml index c0bc3f363f..f97e333f13 100644 --- a/tests/llm/fixtures/test_investigate/03_cpu_throttling/test_case.yaml +++ b/tests/llm/fixtures/test_investigate/03_cpu_throttling/test_case.yaml @@ -1,9 +1,9 @@ expected_output: - The frontend-service pod in the default namespace is experiencing 100% CPU throttling for the stress container - Suggest increasing the CPU limit -before-test: | +before_test: | kubectl apply -f manifest.yaml sleep 60 -after-test: kubectl delete -f manifest.yaml +after_test: kubectl delete -f manifest.yaml evaluation: correctness: 1 From fb3cae59740faac8e4b2fe440be2f2c1ce5947ee Mon Sep 17 00:00:00 2001 From: Mohse Morad Date: Tue, 5 Aug 2025 11:40:06 +0300 Subject: [PATCH 10/11] Fix test type --- .../test_ask_holmes/93_calling_datadog/toolsets.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/toolsets.yaml index 40e92b7eb1..afeb2f408f 100644 --- a/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/toolsets.yaml +++ b/tests/llm/fixtures/test_ask_holmes/93_calling_datadog/toolsets.yaml @@ -9,19 +9,7 @@ toolsets: enabled: true datadog/logs: enabled: True - config: - dd_app_key: "{{ env.DATADOG_APP_KEY }}" - dd_api_key: "{{ env.DATADOG_API_KEY }}" - site_api_url: https://api.datadoghq.eu datadog/metrics: enabled: True - config: - dd_app_key: "{{ env.DATADOG_APP_KEY }}" - dd_api_key: "{{ env.DATADOG_API_KEY }}" - site_api_url: https://api.datadoghq.eu datadog/traces: enabled: True - config: - dd_app_key: "{{ env.DATADOG_APP_KEY }}" - dd_api_key: "{{ env.DATADOG_API_KEY }}" - site_api_url: https://api.datadoghq.eu From 32eb81053b85c244908723d5bdc283902de17c4e Mon Sep 17 00:00:00 2001 From: Mohse Morad Date: Tue, 5 Aug 2025 11:49:23 +0300 Subject: [PATCH 11/11] Fix tests and warnings --- holmes/core/models.py | 1 - tests/llm/utils/test_case_utils.py | 9 +++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/holmes/core/models.py b/holmes/core/models.py index c610fbb252..8eec8a7e4f 100644 --- a/holmes/core/models.py +++ b/holmes/core/models.py @@ -126,7 +126,6 @@ class WorkloadHealthRequest(BaseModel): include_tool_call_results: bool = False prompt_template: str = "builtin://kubernetes_workload_ask.jinja2" model: Optional[str] = None - request: Any = None class ChatRequest(ChatRequestBaseModel): diff --git a/tests/llm/utils/test_case_utils.py b/tests/llm/utils/test_case_utils.py index 90866c1c75..462caa638b 100644 --- a/tests/llm/utils/test_case_utils.py +++ b/tests/llm/utils/test_case_utils.py @@ -7,7 +7,7 @@ from typing import Any, List, Literal, Optional, TypeVar, Union, cast import pytest -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, TypeAdapter, ConfigDict from holmes.core.models import InvestigateRequest, WorkloadHealthRequest from holmes.core.prompt import append_file_to_user_prompt @@ -42,8 +42,7 @@ class Message(BaseModel): class HolmesTestCase(BaseModel): - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") id: str folder: str @@ -71,9 +70,6 @@ class Config: class AskHolmesTestCase(HolmesTestCase, BaseModel): - class Config: - extra = "forbid" - user_prompt: Union[ str, List[str] ] # The user's question(s) to ask holmes - can be single string or array @@ -102,6 +98,7 @@ class HealthCheckTestCase(HolmesTestCase, BaseModel): issue_data: Optional[Dict] resource_instructions: Optional[ResourceInstructions] expected_sections: Optional[Dict[str, Union[List[str], bool]]] = None + request: Any = None def check_and_skip_test(test_case: HolmesTestCase) -> None: