diff --git a/cli/azd/cmd/init.go b/cli/azd/cmd/init.go index b007ae5f8b4..7bec25d86a8 100644 --- a/cli/azd/cmd/init.go +++ b/cli/azd/cmd/init.go @@ -371,6 +371,7 @@ func (i *initAction) Run(ctx context.Context) (*actions.ActionResult, error) { return nil, err } case initEnvironment: + tracing.SetUsageAttributes(fields.InitMethod.String("environment")) env, err := i.initializeEnv(ctx, azdCtx, templates.Metadata{}) if err != nil { return nil, err @@ -379,7 +380,7 @@ func (i *initAction) Run(ctx context.Context) (*actions.ActionResult, error) { header = fmt.Sprintf("Initialized environment %s.", env.Name()) followUp = "" case initWithAgent: - tracing.SetUsageAttributes(fields.InitMethod.String("agent")) + tracing.SetUsageAttributes(fields.InitMethod.String("copilot")) if err := i.initAppWithAgent(ctx, azdCtx); err != nil { return nil, err } @@ -541,9 +542,9 @@ When complete, provide a brief summary of what was accomplished.` } // Show session metrics (usage + file changes) - if metrics := copilotAgent.GetMetrics().String(); metrics != "" { + if metricsStr := copilotAgent.GetMetrics().String(); metricsStr != "" { i.console.Message(ctx, "") - i.console.Message(ctx, metrics) + i.console.Message(ctx, metricsStr) } i.console.Message(ctx, "") diff --git a/cli/azd/internal/agent/consent/workflow_consent.go b/cli/azd/internal/agent/consent/workflow_consent.go index 74649d3073d..c1ef8e23879 100644 --- a/cli/azd/internal/agent/consent/workflow_consent.go +++ b/cli/azd/internal/agent/consent/workflow_consent.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/output" "github.com/azure/azure-dev/cli/azd/pkg/ux" "github.com/mark3labs/mcp-go/mcp" @@ -29,6 +31,13 @@ func (cm *consentManager) PromptWorkflowConsent(ctx context.Context, servers []s return err } + // Track the consent scope selection + if scope == "" { + tracing.SetUsageAttributes(fields.CopilotInitConsentScope.String("prompt")) + } else { + tracing.SetUsageAttributes(fields.CopilotInitConsentScope.String(string(scope))) + } + // Empty scope means the user chose "No, prompt me for each operation" — no rules to add if scope == "" { return nil diff --git a/cli/azd/internal/agent/copilot_agent.go b/cli/azd/internal/agent/copilot_agent.go index ac8a66b37bd..33bf4825a52 100644 --- a/cli/azd/internal/agent/copilot_agent.go +++ b/cli/azd/internal/agent/copilot_agent.go @@ -21,6 +21,9 @@ import ( "github.com/azure/azure-dev/cli/azd/internal/agent/consent" agentcopilot "github.com/azure/azure-dev/cli/azd/internal/agent/copilot" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/events" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/config" "github.com/azure/azure-dev/cli/azd/pkg/input" "github.com/azure/azure-dev/cli/azd/pkg/output" @@ -57,6 +60,9 @@ type CopilotAgent struct { mu sync.Mutex // guards cumulative metrics and file changes cumulativeUsage UsageMetrics // cumulative metrics across multiple SendMessage calls accumulatedFileChanges watch.FileChanges // accumulated file changes across all SendMessage calls + messageCount int // number of messages sent in current session + consentApprovedCount int // running count of tool calls approved + consentDeniedCount int // running count of tool calls denied // Cleanup — ordered slice for deterministic teardown cleanupTasks []cleanupTask @@ -70,7 +76,19 @@ type cleanupTask struct { // Initialize handles first-run configuration (model/reasoning prompts), plugin install, // and Copilot client startup. If config already exists, returns current values without // prompting. Use WithForcePrompt() to always show prompts. -func (a *CopilotAgent) Initialize(ctx context.Context, opts ...InitOption) (*InitResult, error) { +func (a *CopilotAgent) Initialize(ctx context.Context, opts ...InitOption) (result *InitResult, err error) { + ctx, span := tracing.Start(ctx, events.CopilotInitializeEvent) + defer func() { + if result != nil { + tracing.SetUsageAttributes( + fields.CopilotInitIsFirstRun.Bool(result.IsFirstRun), + fields.CopilotInitModel.String(result.Model), + fields.CopilotInitReasoningEffort.String(result.ReasoningEffort), + ) + } + span.EndWithStatus(err) + }() + options := &initOptions{} for _, opt := range opts { opt(options) @@ -298,11 +316,23 @@ func (a *CopilotAgent) SendMessage(ctx context.Context, prompt string, opts ...S log.Printf("[copilot] SendMessage: sending prompt (%d chars, headless=%v)...", len(prompt), a.headless) + var result *AgentResult + var err error + if a.headless { - return a.sendMessageHeadless(ctx, prompt, mode) + result, err = a.sendMessageHeadless(ctx, prompt, mode) + } else { + result, err = a.sendMessageInteractive(ctx, prompt, mode) + } + + if err != nil { + return nil, err } - return a.sendMessageInteractive(ctx, prompt, mode) + // Increment message count only after successful send + a.messageCount++ + + return result, nil } // sendMessageInteractive sends a message with full interactive display and file watcher. @@ -470,6 +500,23 @@ func (a *CopilotAgent) SendMessageWithRetry(ctx context.Context, prompt string, // Stop terminates the agent, cleans up the Copilot SDK client and runtime process. // Cleanup tasks run in reverse registration order. Safe to call multiple times. func (a *CopilotAgent) Stop() error { + // Record all cumulative session metrics as usage attributes + tracing.SetUsageAttributes( + fields.CopilotMode.String(string(a.mode)), + fields.CopilotSessionMessageCount.Int(a.messageCount), + fields.CopilotMessageModel.String(a.cumulativeUsage.Model), + fields.CopilotMessageInputTokens.Float64(a.cumulativeUsage.InputTokens), + fields.CopilotMessageOutputTokens.Float64(a.cumulativeUsage.OutputTokens), + fields.CopilotMessageBillingRate.Float64(a.cumulativeUsage.BillingRate), + fields.CopilotMessagePremiumRequests.Float64(a.cumulativeUsage.PremiumRequests), + fields.CopilotMessageDurationMs.Float64(a.cumulativeUsage.DurationMS), + fields.CopilotConsentApprovedCount.Int(a.consentApprovedCount), + fields.CopilotConsentDeniedCount.Int(a.consentDeniedCount), + ) + if a.sessionID != "" { + tracing.SetUsageAttributes(fields.StringHashed(fields.CopilotSessionId, a.sessionID)) + } + tasks := a.cleanupTasks a.cleanupTasks = nil @@ -538,11 +585,27 @@ func (a *CopilotAgent) EnsureStarted(ctx context.Context) error { // ensureSession creates or resumes a Copilot session if one doesn't exist. // Uses context.WithoutCancel to prevent the SDK session from being torn down // when a per-request context (e.g., gRPC) is cancelled between calls. -func (a *CopilotAgent) ensureSession(ctx context.Context, resumeSessionID string) error { +func (a *CopilotAgent) ensureSession(ctx context.Context, resumeSessionID string) (err error) { if a.session != nil { return nil } + ctx, span := tracing.Start(ctx, events.CopilotSessionEvent) + defer func() { + // Log session ID even on failure (e.g., resume failure still has the attempted ID) + sessionID := a.sessionID + if sessionID == "" && resumeSessionID != "" { + sessionID = resumeSessionID + } + if sessionID != "" { + span.SetAttributes(fields.StringHashed(fields.CopilotSessionId, sessionID)) + } + span.EndWithStatus(err) + }() + + isResume := resumeSessionID != "" + span.SetAttributes(fields.CopilotSessionIsNew.Bool(!isResume)) + // Detach from the caller's cancellation so the client and session // outlive individual requests (e.g., gRPC calls). sessionCtx := context.WithoutCancel(ctx) @@ -647,6 +710,7 @@ func (a *CopilotAgent) createPermissionHandler() copilot.PermissionHandlerFunc { // In headless mode, auto-approve all permission requests if a.headless { log.Printf("[copilot] PermissionRequest (headless auto-approve): kind=%s", req.Kind) + a.consentApprovedCount++ return copilot.PermissionRequestResult{Kind: "approved"}, nil } @@ -667,10 +731,12 @@ func (a *CopilotAgent) createPermissionHandler() copilot.PermissionHandlerFunc { decision, err := a.consentManager.CheckConsent(a.activeContext(), consentReq) if err != nil { log.Printf("[copilot] Consent check error for %s: %v, denying", toolID, err) + a.consentDeniedCount++ return copilot.PermissionRequestResult{Kind: "denied-by-rules"}, nil } if decision.Allowed { + a.consentApprovedCount++ return copilot.PermissionRequestResult{Kind: "approved"}, nil } @@ -693,18 +759,23 @@ func (a *CopilotAgent) createPermissionHandler() copilot.PermissionHandlerFunc { if promptErr != nil { if errors.Is(promptErr, consent.ErrToolExecutionSkipped) { // Skip — deny this tool but let the agent continue + a.consentDeniedCount++ return copilot.PermissionRequestResult{Kind: "denied-by-rules"}, nil } if errors.Is(promptErr, consent.ErrToolExecutionDenied) { // Deny — block and exit the interaction + a.consentDeniedCount++ return copilot.PermissionRequestResult{Kind: "denied-interactively-by-user"}, nil } log.Printf("[copilot] Consent grant error for %s: %v", toolID, promptErr) + a.consentDeniedCount++ return copilot.PermissionRequestResult{Kind: "denied-by-rules"}, nil } + a.consentApprovedCount++ return copilot.PermissionRequestResult{Kind: "approved"}, nil } + a.consentDeniedCount++ return copilot.PermissionRequestResult{Kind: "denied-by-rules"}, nil } } diff --git a/cli/azd/internal/tracing/events/events.go b/cli/azd/internal/tracing/events/events.go index e18a496b8b9..d160d895d7b 100644 --- a/cli/azd/internal/tracing/events/events.go +++ b/cli/azd/internal/tracing/events/events.go @@ -29,3 +29,12 @@ const ( ExtensionRunEvent = "ext.run" ExtensionInstallEvent = "ext.install" ) + +// Copilot agent related events. +const ( + // CopilotInitializeEvent tracks the agent initialization flow (model/reasoning config). + CopilotInitializeEvent = "copilot.initialize" + + // CopilotSessionEvent tracks session creation or resumption. + CopilotSessionEvent = "copilot.session" +) diff --git a/cli/azd/internal/tracing/fields/fields.go b/cli/azd/internal/tracing/fields/fields.go index d06a48e9659..494149a9aac 100644 --- a/cli/azd/internal/tracing/fields/fields.go +++ b/cli/azd/internal/tracing/fields/fields.go @@ -621,3 +621,123 @@ var ( Purpose: FeatureInsight, } ) + +// Copilot agent session related fields +var ( + // CopilotSessionId is the hashed session ID for correlation across messages. + CopilotSessionId = AttributeKey{ + Key: attribute.Key("copilot.session.id"), + Classification: EndUserPseudonymizedInformation, + Purpose: FeatureInsight, + } + // CopilotSessionIsNew indicates whether this was a new session (true) or resumed (false). + CopilotSessionIsNew = AttributeKey{ + Key: attribute.Key("copilot.session.isNew"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // CopilotSessionMessageCount is the number of messages sent in the session. + CopilotSessionMessageCount = AttributeKey{ + Key: attribute.Key("copilot.session.messageCount"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + IsMeasurement: true, + } +) + +// Copilot agent initialization related fields +var ( + // CopilotInitIsFirstRun indicates whether this was the user's first agent initialization. + CopilotInitIsFirstRun = AttributeKey{ + Key: attribute.Key("copilot.init.isFirstRun"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // CopilotInitReasoningEffort is the reasoning level selected (low/medium/high). + CopilotInitReasoningEffort = AttributeKey{ + Key: attribute.Key("copilot.init.reasoningEffort"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // CopilotInitModel is the model ID selected (empty string = default). + CopilotInitModel = AttributeKey{ + Key: attribute.Key("copilot.init.model"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // CopilotInitConsentScope is the workflow consent scope chosen (session/project/global). + CopilotInitConsentScope = AttributeKey{ + Key: attribute.Key("copilot.init.consentScope"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } +) + +// Copilot agent mode and message related fields +var ( + // CopilotMode is the agent operating mode (interactive/autopilot/plan). + CopilotMode = AttributeKey{ + Key: attribute.Key("copilot.mode"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // CopilotMessageModel is the model used for a specific message. + CopilotMessageModel = AttributeKey{ + Key: attribute.Key("copilot.message.model"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // CopilotMessageInputTokens is the number of input tokens consumed per message. + CopilotMessageInputTokens = AttributeKey{ + Key: attribute.Key("copilot.message.inputTokens"), + Classification: SystemMetadata, + Purpose: PerformanceAndHealth, + IsMeasurement: true, + } + // CopilotMessageOutputTokens is the number of output tokens consumed per message. + CopilotMessageOutputTokens = AttributeKey{ + Key: attribute.Key("copilot.message.outputTokens"), + Classification: SystemMetadata, + Purpose: PerformanceAndHealth, + IsMeasurement: true, + } + // CopilotMessageBillingRate is the billing rate multiplier per message. + CopilotMessageBillingRate = AttributeKey{ + Key: attribute.Key("copilot.message.billingRate"), + Classification: SystemMetadata, + Purpose: BusinessInsight, + IsMeasurement: true, + } + // CopilotMessagePremiumRequests is the number of premium requests used per message. + CopilotMessagePremiumRequests = AttributeKey{ + Key: attribute.Key("copilot.message.premiumRequests"), + Classification: SystemMetadata, + Purpose: BusinessInsight, + IsMeasurement: true, + } + // CopilotMessageDurationMs is the API call duration in milliseconds per message. + CopilotMessageDurationMs = AttributeKey{ + Key: attribute.Key("copilot.message.durationMs"), + Classification: SystemMetadata, + Purpose: PerformanceAndHealth, + IsMeasurement: true, + } +) + +// Copilot consent related fields +var ( + // CopilotConsentApprovedCount is the running count of tool calls approved during the session. + CopilotConsentApprovedCount = AttributeKey{ + Key: attribute.Key("copilot.consent.approvedCount"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + IsMeasurement: true, + } + // CopilotConsentDeniedCount is the running count of tool calls denied during the session. + CopilotConsentDeniedCount = AttributeKey{ + Key: attribute.Key("copilot.consent.deniedCount"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + IsMeasurement: true, + } +)