Skip to content

Set up Datadog metrics for prompt injection detection #5385

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dorien-koelemeijer merged 1 commit into from
Oct 27, 2025
Merged

Set up Datadog metrics for prompt injection detection #5385

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions crates/goose/src/agents/tool_execution.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ impl Agent {
// Log user decision if this was a security alert
if let Some(finding_id) = get_security_finding_id_from_results(&request.id, inspection_results) {
tracing::info!(
counter.goose.prompt_injection_user_decisions = 1,
decision = ?confirmation.permission,
"🔒 User security decision: {:?} for finding ID: {}",
confirmation.permission,
finding_id
Expand Down
65 changes: 32 additions & 33 deletions crates/goose/src/security/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,18 @@ impl SecurityManager {
messages: &[Message],
) -> Result<Vec<SecurityResult>> {
if !self.is_prompt_injection_detection_enabled() {
tracing::debug!("🔓 Security scanning disabled - returning empty results");
tracing::debug!(
gauge.goose.prompt_injection_scanner_enabled = 0,
"🔓 Security scanning disabled"
);
return Ok(vec![]);
}

let scanner = self.scanner.get_or_init(|| {
tracing::info!("Security scanner initialized and enabled");
tracing::info!(
gauge.goose.prompt_injection_scanner_enabled = 1,
"🔓 Security scanner initialized and enabled"
);
PromptInjectionScanner::new()
});

Expand All @@ -65,56 +71,48 @@ impl SecurityManager {
);

// Analyze each tool request
for (i, tool_request) in tool_requests.iter().enumerate() {
for tool_request in tool_requests.iter() {
if let Ok(tool_call) = &tool_request.tool_call {
tracing::info!(
tool_name = %tool_call.name,
tool_index = i,
tool_request_id = %tool_request.id,
tool_args = ?tool_call.arguments,
"🔍 Starting security analysis for current tool call"
);

let analysis_result = scanner
.analyze_tool_call_with_context(tool_call, messages)
.await?;

// Get threshold from config - only flag things above threshold
let config_threshold = scanner.get_threshold_from_config();

if analysis_result.is_malicious && analysis_result.confidence > config_threshold {
// Generate a globally unique finding ID for each security finding
if analysis_result.is_malicious {
let above_threshold = analysis_result.confidence > config_threshold;
let finding_id = format!("SEC-{}", Uuid::new_v4().simple());

tracing::warn!(
counter.goose.prompt_injection_finding = 1,
gauge.goose.prompt_injection_confidence_score = analysis_result.confidence,
above_threshold = above_threshold,
tool_name = %tool_call.name,
tool_request_id = %tool_request.id,
confidence = analysis_result.confidence,
explanation = %analysis_result.explanation,
finding_id = %finding_id,
threshold = config_threshold,
"🔒 Current tool call flagged as malicious after security analysis (above threshold)"
);

results.push(SecurityResult {
is_malicious: analysis_result.is_malicious,
confidence: analysis_result.confidence,
explanation: analysis_result.explanation,
should_ask_user: true, // Always ask user for threats above threshold
finding_id,
tool_request_id: tool_request.id.clone(),
});
} else if analysis_result.is_malicious {
tracing::warn!(
tool_name = %tool_call.name,
tool_request_id = %tool_request.id,
confidence = analysis_result.confidence,
explanation = %analysis_result.explanation,
threshold = config_threshold,
"🔒 Security finding below threshold - logged but not blocking execution"
"{}",
if above_threshold {
"🔒 Current tool call flagged as malicious after security analysis (above threshold)"
} else {
"🔒 Security finding below threshold - logged but not blocking execution"
}
);
if above_threshold {
results.push(SecurityResult {
is_malicious: analysis_result.is_malicious,
confidence: analysis_result.confidence,
explanation: analysis_result.explanation,
should_ask_user: true, // Always ask user for threats above threshold
finding_id,
tool_request_id: tool_request.id.clone(),
});
}
} else {
tracing::debug!(
tracing::info!(
tool_name = %tool_call.name,
tool_request_id = %tool_request.id,
confidence = analysis_result.confidence,
Expand All @@ -126,6 +124,7 @@ impl SecurityManager {
}

tracing::info!(
counter.goose.prompt_injection_analysis_performed = 1,
"🔍 Security analysis complete - found {} security issues in current tool requests",
results.len()
);
Expand Down
Loading